Skip to content

Commit 76d015c

Browse files
ncollierncollier
committed
Merge branch 'master' of github.com:ECP-CANDLE/Supervisor
2 parents 8860b5f + d695b3c commit 76d015c

File tree

4 files changed

+91
-4
lines changed

4 files changed

+91
-4
lines changed

‎workflows/p2b1_mlrMBO/swift/cori_workflow3.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ export EMEWS_PROJECT_ROOT=$( cd $( dirname $0 )/.. ; /bin/pwd )
1212
# See README.md for more information
1313

1414
# The directory in the Benchmarks repo containing P2B1
15-
BENCHMARK_DIR=$EMEWS_PROJECT_ROOT/../../../Benchmarks/Pilot2/P2B1
15+
BENCHMARK_DIR="$EMEWS_PROJECT_ROOT/../../../Benchmarks/common"
16+
BENCHMARK_DIR="$BENCHMARK_DIR:$EMEWS_PROJECT_ROOT/../../../Benchmarks/Pilot2/P2B1"
1617

1718
# The number of MPI processes
1819
# Note that 2 processes are reserved for Swift/EMEMS
@@ -83,6 +84,7 @@ EQR=$EMEWS_PROJECT_ROOT/ext/EQ-R
8384

8485
CMD_LINE_ARGS="$* -pp=$PROPOSE_POINTS -mi=$MAX_ITERATIONS -mb=$MAX_BUDGET -ds=$DESIGN_SIZE "
8586
CMD_LINE_ARGS+="-param_set_file=$PARAM_SET_FILE "
87+
CMD_LINE_ARGS+="-exp_id=$EXPID "
8688

8789
# P2B1 requires theano -- doesn't work with tensor flow
8890
KERAS_BACKEND=theano

‎workflows/p2b1_mlrMBO/swift/workflow3.swift‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ int max_budget = toint(argv("mb", "110"));
1717
int max_iterations = toint(argv("mi", "10"));
1818
int design_size = toint(argv("ds", "10"));
1919
string param_set = argv("param_set_file");
20+
string exp_id = argv("exp_id");
2021

2122
string code_template =
2223
"""
@@ -32,11 +33,35 @@ hyper_parameter_map = json.loads('%s')
3233
hyper_parameter_map['framework'] = 'keras'
3334
hyper_parameter_map['save'] = '{}/output'.format(outdir)
3435
hyper_parameter_map['instance_directory'] = outdir
36+
hyper_parameter_map['experiment_id'] = '%s'
37+
hyper_parameter_map['run_id'] = '%s'
3538
3639
3740
validation_loss = p2b1_runner.run(hyper_parameter_map)
3841
""";
3942

43+
string code_log_start =
44+
"""
45+
import exp_logger
46+
47+
parameter_map = {}
48+
parameter_map['pp'] = '%d'
49+
parameter_map['iterations'] = '%d'
50+
parameter_map['params'] = \"\"\"%s\"\"\"
51+
parameter_map['algorithm'] = '%s'
52+
parameter_map['experiment_id'] = '%s'
53+
sys_env = \"\"\"%s\"\"\"
54+
55+
exp_logger.start(parameter_map, sys_env)
56+
""";
57+
58+
string code_log_end =
59+
"""
60+
import exp_logger
61+
62+
exp_logger.end('%s')
63+
""";
64+
4065
// algorithm params format is a string representation
4166
// of a python dictionary. eqpy_hyperopt evals this
4267
// string to create the dictionary. This, unfortunately,
@@ -47,7 +72,7 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
4772

4873
(string obj_result) obj(string params, string iter_indiv_id) {
4974
string outdir = "%s/run_%s" % (turbine_output, iter_indiv_id);
50-
string code = code_template % (outdir, params);
75+
string code = code_template % (outdir, params, exp_id, iter_indiv_id);
5176
//make_dir(outdir) =>
5277
obj_result = python_persist(code, "str(validation_loss)");
5378
printf(obj_result);
@@ -101,6 +126,21 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
101126
}
102127
}
103128

129+
130+
(void o) log_start(string algorithm) {
131+
string ps = join(file_lines(input(param_set)), " ");
132+
string sys_env = join(file_lines(input("%s/turbine.log" % turbine_output)), ", ");
133+
string code = code_log_start % (propose_points, max_iterations, ps, algorithm, exp_id, sys_env);
134+
python_persist(code);
135+
o = propagate();
136+
}
137+
138+
(void o) log_end(){
139+
string code = code_log_end % (exp_id);
140+
python_persist(code);
141+
o = propagate();
142+
}
143+
104144
(void o) start(int ME_rank) {
105145
location ME = locationFromRank(ME_rank);
106146
// TODO: Edit algo_params to include those required by the R
@@ -115,12 +155,14 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
115155
string algo_params = algo_params_template % (max_budget, max_iterations,
116156
design_size, propose_points, param_set);
117157
string algorithm = strcat(emews_root,"/R/mlrMBO3.R");
158+
log_start(algorithm);
118159
EQR_init_script(ME, algorithm) =>
119160
EQR_get(ME) =>
120161
EQR_put(ME, algo_params) =>
121162
loop(ME, ME_rank) => {
122163
EQR_stop(ME) =>
123164
EQR_delete_R(ME);
165+
log_end() =>
124166
o = propagate();
125167
}
126168
}

‎workflows/p3b1_mlrMBO/swift/cori_workflow3.sh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ export EMEWS_PROJECT_ROOT=$( cd $( dirname $0 )/.. ; /bin/pwd )
1212
# See README.md for more information
1313

1414
# The directory in the Benchmarks repo containing P2B1
15-
BENCHMARK_DIR=$EMEWS_PROJECT_ROOT/../../../Benchmarks/Pilot3/P3B1
15+
BENCHMARK_DIR="$EMEWS_PROJECT_ROOT/../../../Benchmarks/common"
16+
BENCHMARK_DIR="$BENCHMARK_DIR:$EMEWS_PROJECT_ROOT/../../../Benchmarks/Pilot3/P3B1"
1617

1718
# The number of MPI processes
1819
# Note that 2 processes are reserved for Swift/EMEMS
@@ -88,6 +89,7 @@ EQR=$EMEWS_PROJECT_ROOT/ext/EQ-R
8889

8990
CMD_LINE_ARGS="$* -pp=$PROPOSE_POINTS -mi=$MAX_ITERATIONS -mb=$MAX_BUDGET -ds=$DESIGN_SIZE "
9091
CMD_LINE_ARGS+="-param_set_file=$PARAM_SET_FILE "
92+
CMD_LINE_ARGS+="-exp_id=$EXPID "
9193

9294
if [ -n "$MACHINE" ]; then
9395
MACHINE="-m $MACHINE"

‎workflows/p3b1_mlrMBO/swift/workflow3.swift‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ int max_budget = toint(argv("mb", "110"));
1717
int max_iterations = toint(argv("mi", "10"));
1818
int design_size = toint(argv("ds", "10"));
1919
string param_set = argv("param_set_file");
20+
string exp_id = argv("exp_id");
2021

2122
string code_template =
2223
"""
@@ -32,10 +33,34 @@ hyper_parameter_map = json.loads('%s')
3233
hyper_parameter_map['framework'] = 'keras'
3334
hyper_parameter_map['save'] = '{}/output'.format(outdir)
3435
hyper_parameter_map['instance_directory'] = outdir
36+
hyper_parameter_map['experiment_id'] = '%s'
37+
hyper_parameter_map['run_id'] = '%s'
3538
3639
validation_loss = p3b1_runner.run(hyper_parameter_map)
3740
""";
3841

42+
string code_log_start =
43+
"""
44+
import exp_logger
45+
46+
parameter_map = {}
47+
parameter_map['pp'] = '%d'
48+
parameter_map['iterations'] = '%d'
49+
parameter_map['params'] = \"\"\"%s\"\"\"
50+
parameter_map['algorithm'] = '%s'
51+
parameter_map['experiment_id'] = '%s'
52+
sys_env = \"\"\"%s\"\"\"
53+
54+
exp_logger.start(parameter_map, sys_env)
55+
""";
56+
57+
string code_log_end =
58+
"""
59+
import exp_logger
60+
61+
exp_logger.end('%s')
62+
""";
63+
3964
// algorithm params format is a string representation
4065
// of a python dictionary. eqpy_hyperopt evals this
4166
// string to create the dictionary. This, unfortunately,
@@ -46,7 +71,7 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
4671

4772
(string obj_result) obj(string params, string iter_indiv_id) {
4873
string outdir = "%s/run_%s" % (turbine_output, iter_indiv_id);
49-
string code = code_template % (outdir, params);
74+
string code = code_template % (outdir, params, exp_id, iter_indiv_id);
5075
//make_dir(outdir) =>
5176
obj_result = python_persist(code, "str(validation_loss)");
5277
printf(obj_result);
@@ -100,6 +125,20 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
100125
}
101126
}
102127

128+
(void o) log_start(string algorithm) {
129+
string ps = join(file_lines(input(param_set)), " ");
130+
string sys_env = join(file_lines(input("%s/turbine.log" % turbine_output)), ", ");
131+
string code = code_log_start % (propose_points, max_iterations, ps, algorithm, exp_id, sys_env);
132+
python_persist(code);
133+
o = propagate();
134+
}
135+
136+
(void o) log_end(){
137+
string code = code_log_end % (exp_id);
138+
python_persist(code);
139+
o = propagate();
140+
}
141+
103142
(void o) start(int ME_rank) {
104143
location ME = locationFromRank(ME_rank);
105144
// TODO: Edit algo_params to include those required by the R
@@ -114,12 +153,14 @@ max.budget = %d, max.iterations = %d, design.size=%d, propose.points=%d, param.s
114153
string algo_params = algo_params_template % (max_budget, max_iterations,
115154
design_size, propose_points, param_set);
116155
string algorithm = strcat(emews_root,"/R/mlrMBO3.R");
156+
log_start(algorithm);
117157
EQR_init_script(ME, algorithm) =>
118158
EQR_get(ME) =>
119159
EQR_put(ME, algo_params) =>
120160
loop(ME, ME_rank) => {
121161
EQR_stop(ME) =>
122162
EQR_delete_R(ME);
163+
log_end() =>
123164
o = propagate();
124165
}
125166
}

0 commit comments

Comments
 (0)