Merge branch 'master' of github.com:ECP-CANDLE/Supervisor

Author: ncollier

workflows/p1b1_grid/data/settings.json

@@ -1,7 +1,9 @@
 {
     "parameters":
           { 
-          	"1": [2,4],
-            "2": [5]
+          	"1": [2,4,6],
+            "2": [15, 25,50,75],
+            "3": [2000, 1000], 
+            "4": [600, 400]
           }
 }

workflows/p1b1_grid/python/evaluateOne.py

@@ -1,6 +1,7 @@
 import sys
-import p1b1_baseline_keras2
-import p1b1
+import p1b1_runner
+import json
+
 
 if (len(sys.argv) < 3):
 	print('requires arg1=param and arg2=filename')
@@ -9,38 +10,27 @@
 parameterString = sys.argv[1]
 filename        = sys.argv[2]
 
-print (parameterString)
-print ("filename is ", filename)
-
-epochs = int(parameterString[0].strip())
-batch_size = int(parameterString[2].strip())
-print ("Running p1b1 for epochs ", epochs, batch_size)
+# print (parameterString)
+print ("filename is " + filename)
 
-# N1 = int(parameterString[2].strip())
-# NE = int(parameterString[3].strip())
 
-print("Set the correct paths for test and train file")
-test_path="/home/jain/Benchmarks/Data/Pilot1/P1B1.test.csv"
-train_path="/home/jain/Benchmarks/Data/Pilot1/P1B1.train.csv"
+integs = [int(x) for x in parameterString.split(',')]
+print (integs)
 
-print ("Starting to loading Xtrain and Xtest")
-X_train, X_test = p1b1.load_data(test_path=test_path, train_path=train_path)
-print ("Done loading Xtrain and Xtest")
-
-print ("Running p1b1 for epochs ", epochs)
-encoder, decoder, history = p1b1_baseline_keras2.run_p1b1(X_train, X_test, epochs=epochs, batch_size=batch_size)
-print ("Done running p1b1 for epochs ", epochs)
+hyper_parameter_map = {'epochs' : integs[0]}
+hyper_parameter_map['framework'] = 'keras'
+hyper_parameter_map['batch_size'] = integs[1]
+hyper_parameter_map['dense'] = [integs[2], integs[3]] 
+hyper_parameter_map['save'] = './output'
 
+val_loss = p1b1_runner.run(hyper_parameter_map)
+print (val_loss)
 # works around this error:
 # https://github.com/tensorflow/tensorflow/issues/3388
 from keras import backend as K
 K.clear_session()
 
-# use the last validation_loss as the value to minimize
-val_loss = history.history['val_loss']
-r = val_loss[-1]
-
 # writing the val loss to the output file
 with open(filename, 'w') as the_file:
-    the_file.write(repr(r))
+    the_file.write(repr(val_loss))
 

workflows/p1b1_grid/python/p1b1_runner.py

@@ -0,0 +1,107 @@
+# tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
+# so we need to create a synthetic argv.
+import sys
+if not hasattr(sys, 'argv'):
+    sys.argv  = ['p1b1']
+
+import json
+import os
+import p1b1
+import numpy as np
+
+DATA_TYPES = {type(np.float16): 'f16', type(np.float32): 'f32', type(np.float64): 'f64'}
+
+def write_params(params, hyper_parameter_map):
+    parent_dir =  hyper_parameter_map['instance_directory'] if 'instance_directory' in hyper_parameter_map else '.'
+    f = "{}/parameters_p1b1.txt".format(parent_dir)
+    with open(f, "w") as f_out:
+        f_out.write("[parameters]\n")
+        for k,v in params.items():
+            if type(v) in DATA_TYPES:
+                v = DATA_TYPES[type(v)]
+            if isinstance(v, basestring):
+                v = "'{}'".format(v)
+            f_out.write("{}={}\n".format(k, v))
+
+def is_numeric(val):
+    try:
+        float(val)
+        return True
+    except ValueError:
+        return False
+
+def format_params(hyper_parameter_map):
+    for k,v in hyper_parameter_map.items():
+        vals = str(v).split(" ")
+        if len(vals) > 1 and is_numeric(vals[0]):
+            # assume this should be a list
+            if "." in vals[0]:
+                hyper_parameter_map[k] = [float(x) for x in vals]
+            else:
+                hyper_parameter_map[k] = [int(x) for x in vals]
+
+
+def run(hyper_parameter_map):
+    framework = hyper_parameter_map['framework']
+    if framework is 'keras':
+        import p1b1_baseline_keras2
+        pkg = p1b1_baseline_keras2
+    elif framework is 'mxnet':
+        import p1b1_baseline_mxnet
+        pkg = p1b1_baseline_mxnet
+    elif framework is 'neon':
+        import p1b1_baseline_neon
+        pkg = p1b1_baseline_neon
+    else:
+        raise ValueError("Invalid framework: {}".format(framework))
+
+    # params is python dictionary
+    sys.argv = ['fail here', '--epochs', '54321']
+    params = pkg.initialize_parameters()
+    format_params(hyper_parameter_map)
+
+    for k,v in hyper_parameter_map.items():
+        #if not k in params:
+        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
+        params[k] = v
+
+    print(params)
+    write_params(params, hyper_parameter_map)
+    history = pkg.run(params)
+
+    if framework is 'keras':
+        # works around this error:
+        # https://github.com/tensorflow/tensorflow/issues/3388
+        try:
+            from keras import backend as K
+            K.clear_session()
+        except AttributeError:      # theano does not have this function
+            pass
+
+    # use the last validation_loss as the value to minimize
+    val_loss = history.history['val_loss']
+    return val_loss[-1]
+
+def write_output(result, instance_directory):
+    with open('{}/result.txt'.format(instance_directory), 'w') as f_out:
+        f_out.write("{}\n".format(result))
+
+def init(param_file, instance_directory):
+    with open(param_file) as f_in:
+        hyper_parameter_map = json.load(f_in)
+
+    hyper_parameter_map['framework'] = 'keras'
+    hyper_parameter_map['save'] = '{}/output'.format(instance_directory)
+    hyper_parameter_map['instance_directory'] = instance_directory
+    
+    return hyper_parameter_map
+
+if __name__ == '__main__':
+    print('p1b1_runner main ' + str(argv))
+    param_file = sys.argv[1]
+    instance_directory = sys.argv[2]
+    hyper_parameter_map = init(param_file, instance_directory)
+    # clear sys.argv so that argparse doesn't object
+    sys.argv = ['p1b1_runner']
+    result = run(hyper_parameter_map)
+    write_output(result, instance_directory)

workflows/p1b1_grid/swift/evaluateOne.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
-filename=result-$1.txt
-python $APP_HOME/../python/evaluateOne.py $1 $filename
+filename=$TURBINE_OUTPUT/result-$1.txt
+python -u $APP_HOME/../python/evaluateOne.py $1 $filename

workflows/p1b1_grid/swift/run-sweep.swift renamed to workflows/p1b1_grid/swift/grid-sweep.swift

@@ -3,40 +3,54 @@ import files;
 import io;
 import sys;
 
-// ===== Interface definitions for the three programs that we call ======
+// ===== Interface definitions for the programs that we call ======
+// Random values are created from bounds specified in data/settings.json file
 app (file f)
 determineParameters(string settingsFilename)
 {
   (getenv("APP_HOME")+"/determineParameters.sh") settingsFilename f;
 }
 
+// This is where the p1b1 runner is called
 app (file f)
 evaluateOne(string params)
 {
   (getenv("APP_HOME")+"/evaluateOne.sh") params f;
 }
 
+// call this to read all the resultsFiles and compute stats
 app ()
 computeStats(string resultsFile)
 {
   (getenv("APP_HOME")+"/computeStats.sh") resultsFile;
 }
 
+// call this to create any required directories
+app (void o) make_dir(string dirname) {
+  "mkdir" "-p" dirname;
+}
+
 
 // ===== The program proper ==============================================
+string turbine_output = getenv("TURBINE_OUTPUT");
 float results[string];
 
+//make the experiments dir
+make_dir(turbine_output);
+
 // Get parameters
 settingsFilename = argv("settings");
-printf(settingsFilename);
-file parametersFile<"parameters.txt"> = determineParameters(settingsFilename);
+string sweepParamFile = turbine_output+"/sweep-parameters.txt";
+file parametersFile<sweepParamFile> = determineParameters(settingsFilename);
 parametersString = read(parametersFile);
 parameters = split(parametersString, ":");
 
 // Run experiments in parallel, passing each a different parameter set
 foreach param in parameters
 {
-    file resultFile<"result-%s.txt"%param> = evaluateOne(param);
+	string rName = turbine_output+"/result-"+param+".txt";
+	printf(rName);
+    file resultFile<rName> = evaluateOne(param);
     results[param] = string2float(read(resultFile));
 }
 
@@ -48,3 +62,4 @@ file tmp = write(repr(results));
 //trace("Temporary filename is: " + filename(tmp));
 
 computeStats(filename(tmp));
+

workflows/p1b1_grid/swift/run

@@ -3,27 +3,42 @@
 # Usage: ./run 
 #
 
-#### set this variable to your P1B1 benchmark directory (supervisor branch)
-P1B1_DIR=/home/jain/newCANDLE/Benchmarks/Pilot1/P1B1
+if [ "$#" -ne 1 ]; then
+  script_name=$(basename $0)
+  echo "Usage: ${script_name} EXPERIMENT_ID (e.g. ${script_name} experiment_1)"
+  exit 1
+fi
+
+#### set this variable to your P1B1 benchmark directory (frameworks branch)
+P1B1_DIR=../../../../Benchmarks/Pilot1/P1B1
 ###
 
 THIS=$( cd $( dirname $0 ); /bin/pwd )
 export APP_HOME=$THIS
 
 PROJECT_ROOT=$APP_HOME/..
 
-### set 
-export PYTHONPATH=/usr/local/lib/python2.7/
-###
-export PYTHONPATH=$PYTHONPATH:$PROJECT_ROOT/python:$P1B1_DIR
+export PYTHONPATH=$PYTHONPATH:$PROJECT_ROOT/python:$P1B1_DIR:$PYTHONPATH
+
+export EXPID=$1
+export TURBINE_OUTPUT=$APP_HOME/../experiments/$EXPID
+
+
+# TODO edit QUEUE, WALLTIME, PPN, AND TURNBINE_JOBNAME
+# as required. Note that QUEUE, WALLTIME, PPN, AND TURNBINE_JOBNAME will
+# be ignored if MACHINE flag (see below) is not set
+export QUEUE=batch
+export WALLTIME=00:10:00
+export PPN=16
+export TURBINE_JOBNAME="${EXPID}_job"
 
 echo $PYTHONPATH
 
 ### set the desired number of processors
-PROCS=3
+PROCS=2
 ###
 
 # remove -l option for removing printing processors ranks
 # settings.json file has all the parameter combinations to be tested
-echo swift-t  -n $PROCS $APP_HOME/run-sweep.swift $*
-swift-t  -l -n $PROCS $APP_HOME/run-sweep.swift $* --settings=$PWD/../data/settings.json
+echo swift-t  -n $PROCS $APP_HOME/grid-sweep.swift $*
+swift-t  -l -n $PROCS $APP_HOME/grid-sweep.swift $* --settings=$PWD/../data/settings.json

workflows/p1b1_random/README.md

@@ -0,0 +1,25 @@
+# Simple parameter sweep with Swift -> parameters randomly chosen between specified bounds.
+The main program (random-sweep.swift) calls a few app functions as follows:
+- determineParameters.{sh,py}: Read data/ **settings.json** for sweep parameters, and return as a string for use by Swift program
+- evaluateOne.{sh,py}: Runs a single experiment. (Calls p1b1_runner).
+- computeStats.{sh,py}: Ingests data from all of the experiments and computes simple stats.
+
+Usage: ./run experient_1
+
+Notes:
+- **settings.json**: 
+A. parameters (benchmark parameters)
+===================================== 
+1: epochs
+2. batch_size
+3. N1
+4. NE
+
+B. samples (specifies the number of random samples to prepare)
+===============================================================
+1. num
+
+For adding new parameters:
+1. Add to the json file the desired parameters
+2. Read params in determineParameters.py: def loadSettings(settingsFilename):
+3. Modify the evaluateOne.py file (set to run on keras framework now)

workflows/p1b1_random/data/settings.json

@@ -0,0 +1,13 @@
+{
+    "parameters":
+          { 
+          	"epochs": [4, 8],
+            "batch_size": [30, 40],
+            "N1": [1000, 2000], 
+            "NE": [400, 600]
+          },
+    "samples":
+    	  {
+    	  	"num": [4]
+    	  }  
+}

workflows/p1b1_random/python/computeStats.py

@@ -0,0 +1,34 @@
+import sys
+from collections import defaultdict
+import json, os
+
+def extractVals(A):
+    B = defaultdict(dict)
+    A1 = A.split()
+    for n, val in zip(A1[0::2], A1[1::2]):
+        B[n] = float(val)
+    return(B)
+
+def computeStats(swiftArrayAsString):
+    A = extractVals(swiftArrayAsString)
+    vals = []
+    for a in A:
+        vals += [A[a]]
+    print('%d values, with min=%f, max=%f, avg=%f\n'%(len(vals),min(vals),max(vals),sum(vals)/float(len(vals))))
+
+
+if (len(sys.argv) < 2):
+	print('requires arg=dataFilename')
+	sys.exit(1)
+
+dataFilename = sys.argv[1]
+
+try:
+    with open(dataFilename, 'r') as the_file:
+        data = the_file.read()
+except IOError as e:
+    print("Could not open: %s" % dataFilename)
+    print("PWD is: '%s'" % os.getcwd())
+
+computeStats(data)
+