o Modify runners and add time for testing all

Author: rajeeja

workflows/rnd_or_grid/python/nt3_tc1_runner.py

@@ -61,10 +61,12 @@ def run(hyper_parameter_map):
     framework = sys.argv[4]
     exp_id = sys.argv[5]
     run_id = sys.argv[6]
+    benchmark_timeout = int(sys.argv[7])
     hyper_parameter_map = runner_utils.init(param_string, instance_directory, framework, 'save')
     hyper_parameter_map['model_name'] = model_name
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
+    hyper_parameter_map['timeout'] = benchmark_timeout
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['nt3_tc1_runner']
     result = run(hyper_parameter_map)

workflows/rnd_or_grid/python/p1b1_runner.py

@@ -7,7 +7,6 @@
 import json
 import os
 import p1b1
-import runner_utils
 
 def run(hyper_parameter_map):
     framework = hyper_parameter_map['framework']
@@ -25,8 +24,6 @@ def run(hyper_parameter_map):
 
     # params is python dictionary
     params = pkg.initialize_parameters()
-    runner_utils.format_params(hyper_parameter_map)
-
     for k,v in hyper_parameter_map.items():
         #if not k in params:
         #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))

workflows/rnd_or_grid/python/p2b1_runner.py

@@ -46,10 +46,12 @@ def run(hyper_parameter_map):
     framework = sys.argv[3]
     exp_id = sys.argv[4]
     run_id = sys.argv[5]
+    benchmark_timeout = int(sys.argv[6])
     hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                             framework, 'save_path')
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
+    hyper_parameter_map['timeout'] = benchmark_timeout
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['p2b1_runner']
     result = run(hyper_parameter_map)

workflows/rnd_or_grid/python/p3b1_runner.py

@@ -8,14 +8,39 @@
 import os
 import p3b1
 import runner_utils
+import socket
+
+node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
+print("node,pid: " + node_pid)
+
+logger = None
+
+def get_logger():
+    """ Set up logging """
+    global logger
+    if logger is not None:
+        return logger
+    import logging, sys
+    logger = logging.getLogger(__name__)
+    logger.setLevel(logging.DEBUG)
+    h = logging.StreamHandler(stream=sys.stdout)
+    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
+                             datefmt='%Y/%m/%d %H:%M:%S')
+    h.setFormatter(fmtr)
+    logger.addHandler(h)
+    return logger
 
 def run(hyper_parameter_map):
+
+    logger = get_logger()
     framework = hyper_parameter_map['framework']
+    logger.debug("IMPORT START")
     if framework == 'keras':
         import p3b1_baseline_keras2
         pkg = p3b1_baseline_keras2
     else:
         raise ValueError("Unsupported framework: {}".format(framework))
+    logger.debug("IMPORT STOP")
 
     # params is python dictionary
     params = pkg.initialize_parameters()
@@ -26,8 +51,12 @@ def run(hyper_parameter_map):
         #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
         params[k] = v
 
+    logger.debug("WRITE_PARAMS START")
     runner_utils.write_params(params, hyper_parameter_map)
+    logger.debug("WRITE_PARAMS STOP")
+    logger.debug("DO_N_FOLD START")
     avg_loss = pkg.do_n_fold(params)
+    logger.debug("DO_N_FOLD STOP")
 
     if framework == 'keras':
         # works around this error:
@@ -41,16 +70,28 @@ def run(hyper_parameter_map):
     return avg_loss
 
 if __name__ == '__main__':
+    logger = get_logger()
+    logger.debug("RUN START")
+
     param_string = sys.argv[1]
     instance_directory = sys.argv[2]
     framework = sys.argv[3]
     exp_id = sys.argv[4]
     run_id = sys.argv[5]
+    benchmark_timeout = int(sys.argv[6])
+
+    logger.debug("RUN INIT START")
+    
     hyper_parameter_map = runner_utils.init(param_string, instance_directory,
                                             framework, 'save_path')
+    logger.debug("RUN INIT STOP")
     hyper_parameter_map['experiment_id'] = exp_id
     hyper_parameter_map['run_id'] = run_id
+    hyper_parameter_map['timeout'] = benchmark_timeout
     # clear sys.argv so that argparse doesn't object
     sys.argv = ['p3b1_runner']
     result = run(hyper_parameter_map)
+    logger.debug("WRITE OUTPUT START")
     runner_utils.write_output(result, instance_directory)
+    logger.debug("WRITE OUTPUT STOP")
+    logger.debug("RUN STOP")

workflows/rnd_or_grid/python/test/test_runners.py

@@ -4,6 +4,8 @@
 import p3b1_runner
 import nt3_tc1_runner
 
+from datetime import datetime
+
 def main():
 
     hyper_parameter_map = {'epochs' : 1}
@@ -14,35 +16,48 @@ def main():
 #     hyper_parameter_map['dense'] = [1219, 536] 
 #     hyper_parameter_map['framework'] = 'keras'
 
-
 #1 # p1b1 - works
 #     hyper_parameter_map['save'] = './p1bl1_testing_failure'
     print("STARTING#####P1B1##########")
+    ts_p1b1 = datetime.now()
     p1b1_validation_loss = p1b1_runner.run(hyper_parameter_map)
-    print("DONE##########P1B1#####")
-
+    te_p1b1 = datetime.now()   
+    print("Validation loss=",p1b1_validation_loss)
+    print("DONE##########P1B1#####, TIME=", te_p1b1 - ts_p1b1)
+    
 
 #2 # p1b3 - works too big for desktop
     print("STARTING#####P1B3##########")
+    te_p1b3 = datetime.now()
     p1b3_validation_loss = p1b3_runner.run(hyper_parameter_map)
-    print("DONE######P1B3#########")
+    te_p1b3 = datetime.now()
+    print("Validation loss=",p1b3_validation_loss)
+    print("DONE##########P1B1#####, TIME=", te_p1b3 - ts_p1b3)
 
 #3 # p2b1 - works
     print("STARTING#####P2B1##########")
+    te_p2b1 = datetime.now()
     p2b1_validation_loss = p2b1_runner.run(hyper_parameter_map)
-    print("DONE#####P2B1##########")
+    te_p2b1 = datetime.now()
+    print("Validation loss=",p2b1_validation_loss)
+    print("DONE##########P1B1#####, TIME=", te_p2b1 - ts_p2b1)
 
 #4 # p3b1 - fails - ValueError: invalid literal for int() with base 10: '1200;1200'
     print("STARTING#####P3B1##########")
+    te_p3b1 = datetime.now()
     p3b1_validation_loss = p3b1_runner.run(hyper_parameter_map)
-    print("DONE#####P3B1##########")
+    te_p3b1 = datetime.now()
+    print("Validation loss=",p3b1_validation_loss)
+    print("DONE##########P1B1#####, TIME=", te_p3b1 - ts_p3b1)
 
 #5 # NT3 - works - too big 
     print("STARTING#####NT3##########")
     hyper_parameter_map['model_name'] = 'nt3'
+    te_nt3 = datetime.now()
     nt3tc1_validation_losss = nt3_tc1_runner.run(hyper_parameter_map)
-    print("DONE#####NT3##########")
+    te_nt3 = datetime.now()
+    print("Validation loss=",nt3tc1_validation_loss)
+    print("DONE##########P1B1#####, TIME=", te_nt3 - ts_nt3)
 
-#     # print("Validation Loss: ", p1b1_validation_loss)
 if __name__ == '__main__':
     main()