Merge branch 'master' of https://github.com/ECP-CANDLE/Supervisor

Author: pbalapra

workflows/common/sh/langs-theta.sh

@@ -0,0 +1,16 @@
+
+# LANGS Theta
+# Language settings for Theta (Swift, Python, R, Tcl, etc.)
+
+TCL=/home/wozniak/Public/sfw/theta/tcl-8.6.1
+export R=/home/wozniak/Public/sfw/theta/R-3.4.0/lib64/R
+export PY=/home/wozniak/Public/sfw/theta/Python-2.7.12
+export LD_LIBRARY_PATH=$PY/lib:$R/lib:$LD_LIBRARY_PATH
+COMMON_DIR=$EMEWS_PROJECT_ROOT/../common/python
+PYTHONPATH=$EMEWS_PROJECT_ROOT/python:$BENCHMARK_DIR:$COMMON_DIR
+PYTHONHOME=/home/wozniak/Public/sfw/theta/Python-2.7.12
+
+# STC=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc
+STC=/projects/Candle_ECP/swift/pyr/stc
+
+export PATH=$STC/bin:$TCL/bin:$PATH

workflows/common/sh/utils.sh

@@ -35,10 +35,13 @@ EOF
   fi
 
   export EXPID=$1
-  if [ $EXPID = "-a" ]; then
+  if [ $EXPID = "-a" ]
+  then
     export TURBINE_OUTPUT_ROOT=$EMEWS_PROJECT_ROOT/experiments
-    export TURBINE_OUTPUT_FORMAT=X%Q
-    EXPID=SWIFT
+    # Creates a X + a unique integer padded to 3 digits: e.g., X023
+    export TURBINE_OUTPUT_FORMAT="X%Q"
+    EXPID="AUTO"
+    shift
   else
     export TURBINE_OUTPUT=$EMEWS_PROJECT_ROOT/experiments/$EXPID
     check_directory_exists

workflows/nt3_mlrMBO/ext/EQ-R/eqr/titan_build_settings.sh

@@ -0,0 +1,34 @@
+module load gcc
+export PATH=$PATH:/ccs/home/wozniak/Public/sfw/swig-3.0.2/bin
+
+# TITAN BUILD SETTINGS
+
+R_HOME=/sw/xk6/r/3.3.2/sles11.3_gnu4.9.3x/lib64/R
+R_INCLUDE=$R_HOME/include
+R_LIB=$R_HOME/lib
+R_INSIDE=$R_HOME/library/RInside
+RCPP=$R_HOME/library/Rcpp
+
+#system-wide tcl
+TCL=/ccs/home/wozniak/Public/sfw/tcl-8.6.2
+TCL_INCLUDE=$TCL/include
+TCL_LIB=$TCL/lib
+TCL_LIBRARY=tcl8.6
+export PATH=$PATH:/ccs/home/wozniak/Public/sfw/tcl-8.6.2/bin
+
+CPPFLAGS=""
+CPPFLAGS+="-I$TCL_INCLUDE "
+CPPFLAGS+="-I$R_INCLUDE "
+CPPFLAGS+="-I$RCPP/include "
+CPPFLAGS+="-I$R_INSIDE/include "
+CXXFLAGS=$CPPFLAGS
+
+LDFLAGS=""
+LDFLAGS+="-L$R_INSIDE/lib -lRInside "
+LDFLAGS+="-L$R_LIB -lR -lRblas "
+LDFLAGS+="-L$TCL_LIB -l$TCL_LIBRARY "
+LDFLAGS+="-Wl,-rpath -Wl,$TCL_LIB "
+LDFLAGS+="-Wl,-rpath -Wl,$R_LIB "
+LDFLAGS+="-Wl,-rpath -Wl,$R_INSIDE/lib"
+
+export CPPFLAGS CXXFLAGS LDFLAGS

workflows/p3b1_mlrMBO/R/mlrMBO3.R

@@ -25,7 +25,8 @@ parallelMap2 <- function(fun, ...,
   else{
     dots <- list(...)
     string_params <- elements_of_lists_to_json(dots[[1L]])
-    print(paste0("parallelMap2 called with list_param: ",string_params))
+    # print(paste0("parallelMap2 called with list_param: ",string_params))
+    # print(paste0("parallelMap2 called with list of length: ", ???)
     OUT_put(string_params)
     string_results = IN_get()
 

workflows/p3b1_mlrMBO/etc/emews_utils.sh

@@ -1,7 +1,8 @@
 log_script() {
+  # usage: Provide shell script for inclusion in log
   SCRIPT_NAME=$(basename $0)
   mkdir -p $TURBINE_OUTPUT
-  LOG_NAME="${TURBINE_OUTPUT}/${SCRIPT_NAME}.log"
+  LOG_NAME="${TURBINE_OUTPUT}/emews.log"
   echo "### VARIABLES ###" > $LOG_NAME
   set +u
   VARS=( "EMEWS_PROJECT_ROOT" "EXPID" "TURBINE_OUTPUT" \
@@ -24,7 +25,7 @@ log_script() {
 
   echo "" >> $LOG_NAME
   echo "## SCRIPT ###" >> $LOG_NAME
-  cat $EMEWS_PROJECT_ROOT/swift/$SCRIPT_NAME >> $LOG_NAME
+  cat $1 >> $LOG_NAME
 }
 
 check_directory_exists() {

workflows/p3b1_mlrMBO/ext/EQ-R/eqr/titan_build_settings.sh

@@ -1,3 +1,4 @@
+# Titan specific settings to build eqr
 module load gcc
 export PATH=$PATH:/ccs/home/wozniak/Public/sfw/swig-3.0.2/bin
 

workflows/p3b1_mlrMBO/python/p3b1_runner.py

@@ -8,14 +8,39 @@
 import os
 import p3b1
 import runner_utils
+import socket
+
+node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
+print("node,pid: " + node_pid)
+
+logger = None
+
+def get_logger():
+    """ Set up logging """
+    global logger
+    if logger is not None:
+        return logger
+    import logging, sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    h = logging.StreamHandler(stream=sys.stdout)
+    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
+                             datefmt='%Y/%m/%d %H:%M:%S')
+    h.setFormatter(fmtr)
+    logger.addHandler(h)
+    return logger
 
 def run(hyper_parameter_map):
+
+    logger = get_logger()
     framework = hyper_parameter_map['framework']
+    logger.debug("IMPORT START")
     if framework == 'keras':
         import p3b1_baseline_keras2
         pkg = p3b1_baseline_keras2
     else:
         raise ValueError("Unsupported framework: {}".format(framework))
+    logger.debug("IMPORT STOP")
 
     # params is python dictionary
     params = pkg.initialize_parameters()
@@ -26,8 +51,12 @@ def run(hyper_parameter_map):
         #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
         params[k] = v
 
+    logger.debug("WRITE_PARAMS START")
     runner_utils.write_params(params, hyper_parameter_map)
+    logger.debug("WRITE_PARAMS STOP")
+    logger.debug("DO_N_FOLD START")
     avg_loss = pkg.do_n_fold(params)
+    logger.debug("DO_N_FOLD STOP")
 
     if framework == 'keras':
         # works around this error:
@@ -41,16 +70,24 @@ def run(hyper_parameter_map):
     return avg_loss
 
 if __name__ == '__main__':
+    logger = get_logger()
+    logger.debug("RUN START")
+
     param_string = sys.argv[1]
     instance_directory = sys.argv[2]
     framework = sys.argv[3]
     exp_id = sys.argv[4]
     run_id = sys.argv[5]
+    logger.debug("RUN INIT START")
     hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                             framework, 'save_path')
+    logger.debug("RUN INIT STOP")
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['p3b1_runner']
     result = run(hyper_parameter_map)
+    logger.debug("WRITE OUTPUT START")
     runner_utils.write_output(result, instance_directory)
+    logger.debug("WRITE OUTPUT STOP")
+    logger.debug("RUN STOP")

workflows/p3b1_mlrMBO/scripts/theta_run_model.sh

@@ -61,18 +61,29 @@ export PYTHONPATH
 arg_array=("$emews_root/python/p3b1_runner.py" "$parameter_string" "$instance_directory" "$framework"  "$exp_id" "$run_id")
 MODEL_CMD="python ${arg_array[@]}"
 
+msg()
+{
+  echo "theta_run_model.sh: $*"
+}
+
 # Turn bash error checking off. This is
 # required to properly handle the model execution return value
 # the optional timeout.
 set +e
-echo $MODEL_CMD
-$TIMEOUT_CMD python "${arg_array[@]}"
 
+# Format and report model parameters
+# as represented on Python command line:
+msg MODEL_CMD: $MODEL_CMD | \
+  tr -d "{}\""            | \
+  tr "," " "              | \
+  fmt -t -w 1
+echo
+$TIMEOUT_CMD $MODEL_CMD
 RES=$?
 if [ "$RES" -ne 0 ]; then
-	if [ "$RES" == 124 ]; then
-    echo "---> Timeout error in $MODEL_CMD"
+  if [ "$RES" == 124 ]; then
+    msg "---> Timeout error in MODEL_CMD"
   else
-	   echo "---> Error in $MODEL_CMD"
+    msg "---> Error in MODEL_CMD"
   fi
 fi

workflows/p3b1_mlrMBO/swift/ai_workflow3.swift

@@ -21,6 +21,8 @@ file model_script = input(argv("script_file"));
 file log_script = input(argv("log_script"));
 string exp_id = argv("exp_id");
 
+printf("TURBINE_OUTPUT: %s", turbine_output);
+
 string FRAMEWORK = "keras";
 
 

workflows/p3b1_mlrMBO/swift/init-theta.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -eu
+
+source $EMEWS_PROJECT_ROOT/etc/emews_utils.sh
+
+log_script $EMEWS_PROJECT_ROOT/swift/theta_workflow.sh
+