Skip to content

Commit 654e861

Browse files
ncollierncollier
committed
Merge branch 'master' of github.com:ECP-CANDLE/Supervisor
2 parents 25bdea6 + 8f5eb50 commit 654e861

File tree

15 files changed

+409
-73
lines changed

15 files changed

+409
-73
lines changed

‎workflows/common/python/runner_utils.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ def init(param_string, instance_directory, framework, out_dir_key):
2222

2323
hyper_parameter_map['framework'] = framework
2424
hyper_parameter_map[out_dir_key] = '{}/output'.format(instance_directory)
25+
hyper_parameter_map['save'] = hyper_parameter_map[out_dir_key]
2526
hyper_parameter_map['instance_directory'] = instance_directory
2627

2728
return hyper_parameter_map

‎workflows/common/sh/langs-theta.sh‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,5 @@
1-
# LANGS THETA
2-
# Language settings for Theta (Python, R, etc.)
3-
# Assumes WORKFLOWS_ROOT is set
4-
# Assumes modules are loaded (cf. modules-cori.sh)
5-
6-
# Python
1+
# LANGS Theta
2+
# Language settings for Theta (Swift, Python, R, Tcl, etc.)
73

84
TCL=/home/wozniak/Public/sfw/theta/tcl-8.6.1
95
export R=/home/wozniak/Public/sfw/theta/R-3.4.0/lib64/R
@@ -13,4 +9,7 @@ COMMON_DIR=$EMEWS_PROJECT_ROOT/../common/python
139
PYTHONPATH=$EMEWS_PROJECT_ROOT/python:$BENCHMARK_DIR:$COMMON_DIR
1410
PYTHONHOME=/home/wozniak/Public/sfw/theta/Python-2.7.12
1511

16-
export PATH=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc/bin:$TCL/bin:$PATH
12+
# STC=/home/wozniak/Public/sfw/theta/swift-t-pyr/stc
13+
STC=/projects/Candle_ECP/swift/pyr/stc
14+
15+
export PATH=$STC/bin:$TCL/bin:$PATH

‎workflows/common/sh/titan_run_logger.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
set -eu
2+
3+
CMD=$1
4+
EMEWS_PROJECT_ROOT=$2
5+
6+
export PYTHONHOME="/sw/xk6/deeplearning/1.0/sles11.3_gnu4.9.3"
7+
PYTHON="$PYTHONHOME/bin/python"
8+
export LD_LIBRARY_PATH="$PYTHONHOME/lib:/sw/xk6/deeplearning/1.0/sles11.3_gnu4.9.3/lib:/sw/xk6/deeplearning/1.0/sles11.3_gnu4.9.3/cuda/lib64:/opt/gcc/4.9.3/snos/lib64:/sw/xk6/r/3.3.2/sles11.3_gnu4.9.3x/lib64/R/lib"
9+
export PATH="$PYTHONHOME/bin:$PATH"
10+
11+
COMMON=$EMEWS_PROJECT_ROOT/../../../Benchmarks/common
12+
PYTHONPATH="$PYTHONHOME/lib/python3.6:$COMMON"
13+
PYTHONPATH+=":$PYTHONHOME/lib/python3.6/site-packages"
14+
export PYTHONPATH
15+
16+
# "start" propose_points, max_iterations, ps, algorithm, exp_id, sys_env
17+
if [ $CMD == "start" ]
18+
then
19+
arg_array=("$EMEWS_PROJECT_ROOT/../common/python/log_runner.py" "$1" "$3" "$4" "$5" "$6" "$7" "$8")
20+
python "${arg_array[@]}"
21+
else
22+
arg_array=("$EMEWS_PROJECT_ROOT/../common/python/log_runner.py" "$1" "$3")
23+
python "${arg_array[@]}"
24+
fi

‎workflows/common/sh/utils.sh‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,13 @@ EOF
3535
fi
3636

3737
export EXPID=$1
38-
if [ $EXPID = "-a" ]; then
38+
if [ $EXPID = "-a" ]
39+
then
3940
export TURBINE_OUTPUT_ROOT=$EMEWS_PROJECT_ROOT/experiments
40-
export TURBINE_OUTPUT_FORMAT=X%Q
41-
EXPID=SWIFT
41+
# Creates a X + a unique integer padded to 3 digits: e.g., X023
42+
export TURBINE_OUTPUT_FORMAT="X%Q"
43+
EXPID="AUTO"
44+
shift
4245
else
4346
export TURBINE_OUTPUT=$EMEWS_PROJECT_ROOT/experiments/$EXPID
4447
check_directory_exists

‎workflows/nt3_mlrMBO/ext/EQ-R/eqr/titan_build_settings.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
module load gcc
2+
export PATH=$PATH:/ccs/home/wozniak/Public/sfw/swig-3.0.2/bin
3+
4+
# TITAN BUILD SETTINGS
5+
6+
R_HOME=/sw/xk6/r/3.3.2/sles11.3_gnu4.9.3x/lib64/R
7+
R_INCLUDE=$R_HOME/include
8+
R_LIB=$R_HOME/lib
9+
R_INSIDE=$R_HOME/library/RInside
10+
RCPP=$R_HOME/library/Rcpp
11+
12+
#system-wide tcl
13+
TCL=/ccs/home/wozniak/Public/sfw/tcl-8.6.2
14+
TCL_INCLUDE=$TCL/include
15+
TCL_LIB=$TCL/lib
16+
TCL_LIBRARY=tcl8.6
17+
export PATH=$PATH:/ccs/home/wozniak/Public/sfw/tcl-8.6.2/bin
18+
19+
CPPFLAGS=""
20+
CPPFLAGS+="-I$TCL_INCLUDE "
21+
CPPFLAGS+="-I$R_INCLUDE "
22+
CPPFLAGS+="-I$RCPP/include "
23+
CPPFLAGS+="-I$R_INSIDE/include "
24+
CXXFLAGS=$CPPFLAGS
25+
26+
LDFLAGS=""
27+
LDFLAGS+="-L$R_INSIDE/lib -lRInside "
28+
LDFLAGS+="-L$R_LIB -lR -lRblas "
29+
LDFLAGS+="-L$TCL_LIB -l$TCL_LIBRARY "
30+
LDFLAGS+="-Wl,-rpath -Wl,$TCL_LIB "
31+
LDFLAGS+="-Wl,-rpath -Wl,$R_LIB "
32+
LDFLAGS+="-Wl,-rpath -Wl,$R_INSIDE/lib"
33+
34+
export CPPFLAGS CXXFLAGS LDFLAGS

‎workflows/p3b1_mlrMBO/R/mlrMBO3.R‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ parallelMap2 <- function(fun, ...,
2525
else{
2626
dots <- list(...)
2727
string_params <- elements_of_lists_to_json(dots[[1L]])
28-
print(paste0("parallelMap2 called with list_param: ",string_params))
28+
# print(paste0("parallelMap2 called with list_param: ",string_params))
29+
# print(paste0("parallelMap2 called with list of length: ", ???)
2930
OUT_put(string_params)
3031
string_results = IN_get()
3132

‎workflows/p3b1_mlrMBO/etc/emews_utils.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
log_script() {
2+
# usage: Provide shell script for inclusion in log
23
SCRIPT_NAME=$(basename $0)
34
mkdir -p $TURBINE_OUTPUT
4-
LOG_NAME="${TURBINE_OUTPUT}/${SCRIPT_NAME}.log"
5+
LOG_NAME="${TURBINE_OUTPUT}/emews.log"
56
echo "### VARIABLES ###" > $LOG_NAME
67
set +u
78
VARS=( "EMEWS_PROJECT_ROOT" "EXPID" "TURBINE_OUTPUT" \
@@ -24,7 +25,7 @@ log_script() {
2425

2526
echo "" >> $LOG_NAME
2627
echo "## SCRIPT ###" >> $LOG_NAME
27-
cat $EMEWS_PROJECT_ROOT/swift/$SCRIPT_NAME >> $LOG_NAME
28+
cat $1 >> $LOG_NAME
2829
}
2930

3031
check_directory_exists() {

‎workflows/p3b1_mlrMBO/ext/EQ-R/eqr/titan_build_settings.sh‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Titan specific settings to build eqr
2+
module load gcc
3+
export PATH=$PATH:/ccs/home/wozniak/Public/sfw/swig-3.0.2/bin
4+
5+
R_HOME=/sw/xk6/r/3.3.2/sles11.3_gnu4.9.3x/lib64/R
6+
R_INCLUDE=$R_HOME/include
7+
R_LIB=$R_HOME/lib
8+
R_INSIDE=$R_HOME/library/RInside
9+
RCPP=$R_HOME/library/Rcpp
10+
11+
#system-wide tcl
12+
TCL_INCLUDE=/ccs/home/wozniak/Public/sfw/tcl-8.6.2
13+
TCL_LIB=/ccs/home/wozniak/Public/sfw/tcl-8.6.2/lib
14+
TCL_LIBRARY=tcl8.6
15+
export PATH=$PATH:/ccs/home/wozniak/Public/sfw/tcl-8.6.2/bin
16+
17+
CPPFLAGS=""
18+
CPPFLAGS+="-I$TCL_INCLUDE "
19+
CPPFLAGS+="-I$R_INCLUDE "
20+
CPPFLAGS+="-I$RCPP/include "
21+
CPPFLAGS+="-I$R_INSIDE/include "
22+
CXXFLAGS=$CPPFLAGS
23+
24+
LDFLAGS=""
25+
LDFLAGS+="-L$R_INSIDE/lib -lRInside "
26+
LDFLAGS+="-L$R_LIB -lR -lRblas "
27+
LDFLAGS+="-L$TCL_LIB -l$TCL_LIBRARY "
28+
LDFLAGS+="-Wl,-rpath -Wl,$TCL_LIB "
29+
LDFLAGS+="-Wl,-rpath -Wl,$R_LIB "
30+
LDFLAGS+="-Wl,-rpath -Wl,$R_INSIDE/lib"
31+
32+
export CPPFLAGS CXXFLAGS LDFLAGS

‎workflows/p3b1_mlrMBO/python/p3b1_runner.py‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,39 @@
88
import os
99
import p3b1
1010
import runner_utils
11+
import socket
12+
13+
node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
14+
print("node,pid: " + node_pid)
15+
16+
logger = None
17+
18+
def get_logger():
19+
""" Set up logging """
20+
global logger
21+
if logger is not None:
22+
return logger
23+
import logging, sys
24+
logger = logging.getLogger(__name__)
25+
logger.setLevel(logging.DEBUG)
26+
h = logging.StreamHandler(stream=sys.stdout)
27+
fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
28+
datefmt='%Y/%m/%d %H:%M:%S')
29+
h.setFormatter(fmtr)
30+
logger.addHandler(h)
31+
return logger
1132

1233
def run(hyper_parameter_map):
34+
35+
logger = get_logger()
1336
framework = hyper_parameter_map['framework']
37+
logger.debug("IMPORT START")
1438
if framework == 'keras':
1539
import p3b1_baseline_keras2
1640
pkg = p3b1_baseline_keras2
1741
else:
1842
raise ValueError("Unsupported framework: {}".format(framework))
43+
logger.debug("IMPORT STOP")
1944

2045
# params is python dictionary
2146
params = pkg.initialize_parameters()
@@ -26,8 +51,12 @@ def run(hyper_parameter_map):
2651
# raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
2752
params[k] = v
2853

54+
logger.debug("WRITE_PARAMS START")
2955
runner_utils.write_params(params, hyper_parameter_map)
56+
logger.debug("WRITE_PARAMS STOP")
57+
logger.debug("DO_N_FOLD START")
3058
avg_loss = pkg.do_n_fold(params)
59+
logger.debug("DO_N_FOLD STOP")
3160

3261
if framework == 'keras':
3362
# works around this error:
@@ -41,16 +70,24 @@ def run(hyper_parameter_map):
4170
return avg_loss
4271

4372
if __name__ == '__main__':
73+
logger = get_logger()
74+
logger.debug("RUN START")
75+
4476
param_string = sys.argv[1]
4577
instance_directory = sys.argv[2]
4678
framework = sys.argv[3]
4779
exp_id = sys.argv[4]
4880
run_id = sys.argv[5]
81+
logger.debug("RUN INIT START")
4982
hyper_parameter_map = runner_utils.init(param_string, instance_directory,
5083
framework, 'save_path')
84+
logger.debug("RUN INIT STOP")
5185
hyper_parameter_map['experiment_id'] = exp_id
5286
hyper_parameter_map['run_id'] = run_id
5387
# clear sys.argv so that argparse doesn't object
5488
sys.argv = ['p3b1_runner']
5589
result = run(hyper_parameter_map)
90+
logger.debug("WRITE OUTPUT START")
5691
runner_utils.write_output(result, instance_directory)
92+
logger.debug("WRITE OUTPUT STOP")
93+
logger.debug("RUN STOP")

‎workflows/p3b1_mlrMBO/scripts/theta_run_model.sh‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,18 +61,29 @@ export PYTHONPATH
6161
arg_array=("$emews_root/python/p3b1_runner.py" "$parameter_string" "$instance_directory" "$framework" "$exp_id" "$run_id")
6262
MODEL_CMD="python ${arg_array[@]}"
6363

64+
msg()
65+
{
66+
echo "theta_run_model.sh: $*"
67+
}
68+
6469
# Turn bash error checking off. This is
6570
# required to properly handle the model execution return value
6671
# the optional timeout.
6772
set +e
68-
echo $MODEL_CMD
69-
$TIMEOUT_CMD python "${arg_array[@]}"
7073

74+
# Format and report model parameters
75+
# as represented on Python command line:
76+
msg MODEL_CMD: $MODEL_CMD | \
77+
tr -d "{}\"" | \
78+
tr "," " " | \
79+
fmt -t -w 1
80+
echo
81+
$TIMEOUT_CMD $MODEL_CMD
7182
RES=$?
7283
if [ "$RES" -ne 0 ]; then
73-
if [ "$RES" == 124 ]; then
74-
echo "---> Timeout error in $MODEL_CMD"
84+
if [ "$RES" == 124 ]; then
85+
msg "---> Timeout error in MODEL_CMD"
7586
else
76-
echo "---> Error in $MODEL_CMD"
87+
msg "---> Error in MODEL_CMD"
7788
fi
7889
fi

0 commit comments

Comments
 (0)