Merge branch 'master' of https://github.com/ECP-CANDLE/Supervisor into rajeeja/rnd_grid

Author: rajeeja

workflows/common/sh/langs-theta-run-model.sh

@@ -0,0 +1,12 @@
+
+export PYTHONHOME="/lus/theta-fs0/projects/Candle_ECP/ncollier/py2_tf_gcc6.3_eigen3_native"
+PYTHON="$PYTHONHOME/bin/python"
+export LD_LIBRARY_PATH="$PYTHONHOME/lib"
+export PATH="$PYTHONHOME/bin:$PATH"
+
+
+COMMON_DIR=$emews_root/../common/python
+PYTHONPATH="$PYTHONHOME/lib/python2.7:"
+PYTHONPATH+="$BENCHMARK_DIR:$COMMON_DIR:"
+PYTHONPATH+="$PYTHONHOME/lib/python2.7/site-packages"
+export PYTHONPATH

workflows/common/sh/langs-theta.sh

@@ -0,0 +1,15 @@
+# LANGS Theta
+# Language settings for Theta (Swift, Python, R, Tcl, etc.)
+
+TCL=/home/wozniak/Public/sfw/theta/tcl-8.6.1
+export R=/home/wozniak/Public/sfw/theta/R-3.4.0/lib64/R
+export PY=/home/wozniak/Public/sfw/theta/Python-2.7.12
+export LD_LIBRARY_PATH=$PY/lib:$R/lib:$LD_LIBRARY_PATH
+COMMON_DIR=$EMEWS_PROJECT_ROOT/../common/python
+PYTHONPATH=$EMEWS_PROJECT_ROOT/python:$BENCHMARK_DIR:$COMMON_DIR
+PYTHONHOME=/home/wozniak/Public/sfw/theta/Python-2.7.12
+
+# STC=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc
+STC=/projects/Candle_ECP/swift/pyr/stc
+
+export PATH=$STC/bin:$TCL/bin:$PATH

workflows/common/sh/utils.sh

@@ -35,10 +35,13 @@ EOF
   fi
 
   export EXPID=$1
-  if [ $EXPID = "-a" ]; then
+  if [ $EXPID = "-a" ]
+  then
     export TURBINE_OUTPUT_ROOT=$EMEWS_PROJECT_ROOT/experiments
-    export TURBINE_OUTPUT_FORMAT=X%Q
-    EXPID=SWIFT
+    # Creates a X + a unique integer padded to 3 digits: e.g., X023
+    export TURBINE_OUTPUT_FORMAT="X%Q"
+    EXPID="AUTO"
+    shift
   else
     export TURBINE_OUTPUT=$EMEWS_PROJECT_ROOT/experiments/$EXPID
     check_directory_exists

workflows/nt3_mlrMBO/ext/EQ-R/eqr/titan_build_settings.sh

@@ -0,0 +1,34 @@
+module load gcc
+export PATH=$PATH:/ccs/home/wozniak/Public/sfw/swig-3.0.2/bin
+
+# TITAN BUILD SETTINGS
+
+R_HOME=/sw/xk6/r/3.3.2/sles11.3_gnu4.9.3x/lib64/R
+R_INCLUDE=$R_HOME/include
+R_LIB=$R_HOME/lib
+R_INSIDE=$R_HOME/library/RInside
+RCPP=$R_HOME/library/Rcpp
+
+#system-wide tcl
+TCL=/ccs/home/wozniak/Public/sfw/tcl-8.6.2
+TCL_INCLUDE=$TCL/include
+TCL_LIB=$TCL/lib
+TCL_LIBRARY=tcl8.6
+export PATH=$PATH:/ccs/home/wozniak/Public/sfw/tcl-8.6.2/bin
+
+CPPFLAGS=""
+CPPFLAGS+="-I$TCL_INCLUDE "
+CPPFLAGS+="-I$R_INCLUDE "
+CPPFLAGS+="-I$RCPP/include "
+CPPFLAGS+="-I$R_INSIDE/include "
+CXXFLAGS=$CPPFLAGS
+
+LDFLAGS=""
+LDFLAGS+="-L$R_INSIDE/lib -lRInside "
+LDFLAGS+="-L$R_LIB -lR -lRblas "
+LDFLAGS+="-L$TCL_LIB -l$TCL_LIBRARY "
+LDFLAGS+="-Wl,-rpath -Wl,$TCL_LIB "
+LDFLAGS+="-Wl,-rpath -Wl,$R_LIB "
+LDFLAGS+="-Wl,-rpath -Wl,$R_INSIDE/lib"
+
+export CPPFLAGS CXXFLAGS LDFLAGS

workflows/nt3_mlrMBO/python/nt3_tc1_runner.py

@@ -61,10 +61,12 @@ def run(hyper_parameter_map):
     framework = sys.argv[4]
     exp_id = sys.argv[5]
     run_id = sys.argv[6]
+    benchmark_timeout = int(sys.argv[7])
     hyper_parameter_map = runner_utils.init(param_string, instance_directory, framework, 'save')
     hyper_parameter_map['model_name'] = model_name
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
+    hyper_parameter_map['timeout'] = benchmark_timeout
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['nt3_tc1_runner']
     result = run(hyper_parameter_map)

workflows/nt3_mlrMBO/scripts/run_model.sh

@@ -13,7 +13,7 @@ set -eu
 
 # !!! IF YOU CHANGE THE NUMBER OF ARGUMENTS PASSED TO THIS SCRIPT, YOU MUST
 # CHANGE THE TIMEOUT_ARG_INDEX !!!
-TIMEOUT_ARG_INDEX=8
+TIMEOUT_ARG_INDEX=9
 TIMEOUT=""
 if [[ $# ==  $TIMEOUT_ARG_INDEX ]]
 then
@@ -42,12 +42,13 @@ model_name=$4
 framework=$5
 exp_id=$6
 run_id=$7
+benchmark_timeout=$8
 
 BENCHMARK_DIR=$emews_root/../../../Benchmarks/common:$emews_root/../../../Benchmarks/Pilot1/NT3:$emews_root/../../../Benchmarks/Pilot1/TC1
 COMMON_DIR=$emews_root/../common/python
 export PYTHONPATH="$PYTHONPATH:$BENCHMARK_DIR:$COMMON_DIR"
 
-arg_array=("$emews_root/python/nt3_tc1_runner.py" "$parameter_string" "$instance_directory" "$model_name" "$framework" "$exp_id" "$run_id")
+arg_array=("$emews_root/python/nt3_tc1_runner.py" "$parameter_string" "$instance_directory" "$model_name" "$framework" "$exp_id" "$run_id" "$benchmark_timeout")
 MODEL_CMD="python ${arg_array[@]}"
 # Turn bash error checking off. This is
 # required to properly handle the model execution return value

workflows/nt3_mlrMBO/scripts/theta_run_model.sh

@@ -13,7 +13,7 @@ set -eu
 
 # !!! IF YOU CHANGE THE NUMBER OF ARGUMENTS PASSED TO THIS SCRIPT, YOU MUST
 # CHANGE THE TIMEOUT_ARG_INDEX !!!
-TIMEOUT_ARG_INDEX=8
+TIMEOUT_ARG_INDEX=9
 TIMEOUT=""
 if [[ $# ==  $TIMEOUT_ARG_INDEX ]]
 then
@@ -41,6 +41,7 @@ model_name=$4
 framework=$5
 exp_id=$6
 run_id=$7
+benchmark_timeout=$8
 
 # Theta / Tensorflow env vars
 export KMP_BLOCKTIME=30
@@ -60,7 +61,7 @@ PYTHONPATH+="$BENCHMARK_DIR:$COMMON_DIR:"
 PYTHONPATH+="$PYTHONHOME/lib/python2.7/site-packages"
 export PYTHONPATH
 
-arg_array=("$emews_root/python/nt3_tc1_runner.py" "$parameter_string" "$instance_directory" "$model_name" "$framework"  "$exp_id" "$run_id")
+arg_array=("$emews_root/python/nt3_tc1_runner.py" "$parameter_string" "$instance_directory" "$model_name" "$framework"  "$exp_id" "$run_id" "$benchmark_timeout")
 MODEL_CMD="python ${arg_array[@]}"
 
 # Turn bash error checking off. This is

workflows/nt3_mlrMBO/swift/ai_workflow.sh

@@ -26,12 +26,15 @@ export PPN=${PPN:-1}
 export QUEUE=${QUEUE:-debug}
 export WALLTIME=${WALLTIME:-00:30:00}
 
+# Benchmark run timeout: benchmark run will timeouT
+# after the specified number of seconds. -1 is no timeout.
+BENCHMARK_TIMEOUT=${BENCHMARK_TIMEOUT:-3600}
+
 # set machine to your scheduler type (e.g. pbs, slurm, cobalt etc.),
 # or empty for an immediate non-queued unscheduled run
 MACHINE=""
 
 # mlrMBO settings
-# How many to runs evaluate per iteration
 MAX_BUDGET=${MAX_BUDGET:-110}
 # Total iterations
 MAX_ITERATIONS=${MAX_ITERATIONS:-4}
@@ -83,6 +86,7 @@ MODEL_NAME="nt3"
 CMD_LINE_ARGS="$* -pp=$PROPOSE_POINTS -mi=$MAX_ITERATIONS -mb=$MAX_BUDGET -ds=$DESIGN_SIZE "
 CMD_LINE_ARGS+="-param_set_file=$PARAM_SET_FILE -script_file=$SCRIPT_FILE -model_name=$MODEL_NAME "
 CMD_LINE_ARGS+="-exp_id=$EXPID -log_script=$LOG_SCRIPT_FILE "
+CMD_LINE_ARGS+="-benchmark_timeout=$BENCHMARK_TIMEOUT"
 
 if [ -n "$MACHINE" ]; then
   MACHINE="-m $MACHINE"

workflows/nt3_mlrMBO/swift/ai_workflow3.swift

@@ -21,6 +21,7 @@ string model_name = argv("model_name");
 file model_script = input(argv("script_file"));
 file log_script = input(argv("log_script"));
 string exp_id = argv("exp_id");
+int benchmark_timeout = toint(argv("benchmark_timeout", "-1"));
 
 string FRAMEWORK = "keras";
 
@@ -31,7 +32,7 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
 
 app (file out, file err) run_model (file shfile, string params_string, string instance, string run_id)
 {
-    "bash" shfile params_string emews_root instance model_name FRAMEWORK exp_id run_id @stdout=out @stderr=err;
+    "bash" shfile params_string emews_root instance model_name FRAMEWORK exp_id run_id benchmark_timeout @stdout=out @stderr=err;
 }
 
 

workflows/nt3_mlrMBO/swift/cori_workflow3.sh

@@ -1,9 +1,6 @@
 #! /usr/bin/env bash
 set -eu
 
-# CORI WORKFLOW
-# Main entry point for P1B3 mlrMBO workflow
-
 # Autodetect this workflow directory
 export EMEWS_PROJECT_ROOT=$( cd $( dirname $0 )/.. ; /bin/pwd )
 
@@ -28,10 +25,11 @@ export PPN=${PPN:-1}
 export QUEUE=${QUEUE:-regular}
 export WALLTIME=${WALLTIME:-01:00:00}
 
-# mlrMBO settings
-# How many to runs evaluate per iteration
-
+# Benchmark run timeout: benchmark run will timeouT
+# after the specified number of seconds. -1 is no timeout.
+BENCHMARK_TIMEOUT=${BENCHMARK_TIMEOUT:-3600}
 
+# mlrMBO settings
 MAX_BUDGET=${MAX_BUDGET:-1000}
 # Total iterations
 MAX_ITERATIONS=${MAX_ITERATIONS:-4}
@@ -88,6 +86,7 @@ MODEL_NAME="nt3"
 CMD_LINE_ARGS="$* -pp=$PROPOSE_POINTS -mi=$MAX_ITERATIONS -mb=$MAX_BUDGET -ds=$DESIGN_SIZE "
 CMD_LINE_ARGS+="-param_set_file=$PARAM_SET_FILE -model_name=$MODEL_NAME "
 CMD_LINE_ARGS+="-exp_id=$EXPID "
+CMD_LINE_ARGS+="-benchmark_timeout=$BENCHMARK_TIMEOUT"
 
 # set machine to your scheduler type (e.g. pbs, slurm, cobalt etc.),
 # or empty for an immediate non-queued unscheduled run
@@ -113,4 +112,3 @@ WORKFLOW_SWIFT=workflow3.swift
 swift-t -n $PROCS $MACHINE -p -I $EQR -r $EQR \
         -e LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$R_LIB:$GCC_LIB \
         $EMEWS_PROJECT_ROOT/swift/$WORKFLOW_SWIFT $CMD_LINE_ARGS
-