o Adding p1b1_random works with Benchmark(frameworks branch)

o Need to add scripts for running on theta, cooley etc.

Author: rajeeja

workflows/p1b1_random/README.md

@@ -0,0 +1,25 @@
+# Simple parameter sweep with Swift -> parameters randomly chosen between specified bounds.
+The main program (random-sweep.swift) calls a few app functions as follows:
+- determineParameters.{sh,py}: Read data/ **settings.json** for sweep parameters, and return as a string for use by Swift program
+- evaluateOne.{sh,py}: Runs a single experiment. (Calls p1b1_runner).
+- computeStats.{sh,py}: Ingests data from all of the experiments and computes simple stats.
+
+Usage: ./run experient_1
+
+Notes:
+- **settings.json**: 
+A. parameters (benchmark parameters)
+===================================== 
+1: epochs
+2. batch_size
+3. N1
+4. NE
+
+B. samples (specifies the number of random samples to prepare)
+===============================================================
+1. num
+
+For adding new parameters:
+1. Add to the json file the desired parameters
+2. Read params in determineParameters.py: def loadSettings(settingsFilename):
+3. Modify the evaluateOne.py file (set to run on keras framework now)

workflows/p1b1_random/data/settings.json

@@ -0,0 +1,13 @@
+{
+    "parameters":
+          { 
+          	"epochs": [4, 8],
+            "batch_size": [30, 40],
+            "N1": [1000, 2000], 
+            "NE": [400, 600]
+          },
+    "samples":
+    	  {
+    	  	"num": [4]
+    	  }  
+}

workflows/p1b1_random/python/computeStats.py

@@ -0,0 +1,34 @@
+import sys
+from collections import defaultdict
+import json, os
+
+def extractVals(A):
+    B = defaultdict(dict)
+    A1 = A.split()
+    for n, val in zip(A1[0::2], A1[1::2]):
+        B[n] = float(val)
+    return(B)
+
+def computeStats(swiftArrayAsString):
+    A = extractVals(swiftArrayAsString)
+    vals = []
+    for a in A:
+        vals += [A[a]]
+    print('%d values, with min=%f, max=%f, avg=%f\n'%(len(vals),min(vals),max(vals),sum(vals)/float(len(vals))))
+
+
+if (len(sys.argv) < 2):
+	print('requires arg=dataFilename')
+	sys.exit(1)
+
+dataFilename = sys.argv[1]
+
+try:
+    with open(dataFilename, 'r') as the_file:
+        data = the_file.read()
+except IOError as e:
+    print("Could not open: %s" % dataFilename)
+    print("PWD is: '%s'" % os.getcwd())
+
+computeStats(data)
+

workflows/p1b1_random/python/determineParameters.py

@@ -0,0 +1,57 @@
+import sys, json, os
+from random import randint
+
+# ===== Definitions =========================================================
+
+def loadSettings(settingsFilename):
+    print("Reading settings: %s" % settingsFilename)
+    try:
+        with open(settingsFilename) as fp:
+            settings = json.load(fp)
+    except IOError as e:
+        print("Could not open: %s" % settingsFilename)
+        print("PWD is: '%s'" % os.getcwd())
+        sys.exit(1)
+    try:
+        epochs = settings['parameters']["epochs"]
+        batch_size = settings['parameters']["batch_size"]
+        N1 = settings['parameters']["N1"]
+        NE = settings['parameters']["NE"]        
+
+    except KeyError as e:
+        print("Settings file (%s) does not contain key: %s" % (settingsFilename, str(e)))
+        sys.exit(1)
+    try:
+        samples = settings['samples']["num"]
+    except KeyError as e:
+        print("Settings file (%s) does not contain key: %s" % (settingsFilename, str(e)))
+        sys.exit(1)
+    return(epochs, batch_size, N1, NE, samples)
+
+# ===== Main program ========================================================
+
+if (len(sys.argv) < 3):
+	print('requires arg1=settingsFilename and arg2=paramsFilename')
+	sys.exit(1)
+
+settingsFilename = sys.argv[1]
+paramsFilename   = sys.argv[2]
+
+print (settingsFilename)
+print (paramsFilename)
+
+epochs, batch_size, N1, NE, samples = loadSettings(settingsFilename)
+result=""
+
+for s in range(samples[0]):
+    t_epoch= randint(epochs[0], epochs[1])
+    t_batch_size= randint(batch_size[0], batch_size[1])
+    t_N1= randint(N1[0], N1[1])
+    t_NE= randint(NE[0], NE[1])
+    result+=str(t_epoch) + ',' + str(t_batch_size) + ',' + str(t_N1) + ',' + str(t_NE) 
+    if(s < (samples[0]-1)):
+        result+=":"
+
+with open(paramsFilename, 'w') as the_file:
+    the_file.write(result)
+

workflows/p1b1_random/python/evaluateOne.py

@@ -0,0 +1,36 @@
+import sys
+import p1b1_runner
+import json
+
+
+if (len(sys.argv) < 3):
+	print('requires arg1=param and arg2=filename')
+	sys.exit(1)
+
+parameterString = sys.argv[1]
+filename        = sys.argv[2]
+
+# print (parameterString)
+print ("filename is " + filename)
+
+
+integs = [int(x) for x in parameterString.split(',')]
+print (integs)
+
+hyper_parameter_map = {'epochs' : integs[0]}
+hyper_parameter_map['framework'] = 'keras'
+hyper_parameter_map['batch_size'] = integs[1]
+hyper_parameter_map['dense'] = [integs[2], integs[3]] 
+hyper_parameter_map['save'] = './output'
+
+val_loss = p1b1_runner.run(hyper_parameter_map)
+print (val_loss)
+# works around this error:
+# https://github.com/tensorflow/tensorflow/issues/3388
+from keras import backend as K
+K.clear_session()
+
+# writing the val loss to the output file
+with open(filename, 'w') as the_file:
+    the_file.write(repr(val_loss))
+

workflows/p1b1_random/python/p1b1_runner.py

@@ -0,0 +1,107 @@
+# tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
+# so we need to create a synthetic argv.
+import sys
+if not hasattr(sys, 'argv'):
+    sys.argv  = ['p1b1']
+
+import json
+import os
+import p1b1
+import numpy as np
+
+DATA_TYPES = {type(np.float16): 'f16', type(np.float32): 'f32', type(np.float64): 'f64'}
+
+def write_params(params, hyper_parameter_map):
+    parent_dir =  hyper_parameter_map['instance_directory'] if 'instance_directory' in hyper_parameter_map else '.'
+    f = "{}/parameters_p1b1.txt".format(parent_dir)
+    with open(f, "w") as f_out:
+        f_out.write("[parameters]\n")
+        for k,v in params.items():
+            if type(v) in DATA_TYPES:
+                v = DATA_TYPES[type(v)]
+            if isinstance(v, basestring):
+                v = "'{}'".format(v)
+            f_out.write("{}={}\n".format(k, v))
+
+def is_numeric(val):
+    try:
+        float(val)
+        return True
+    except ValueError:
+        return False
+
+def format_params(hyper_parameter_map):
+    for k,v in hyper_parameter_map.items():
+        vals = str(v).split(" ")
+        if len(vals) > 1 and is_numeric(vals[0]):
+            # assume this should be a list
+            if "." in vals[0]:
+                hyper_parameter_map[k] = [float(x) for x in vals]
+            else:
+                hyper_parameter_map[k] = [int(x) for x in vals]
+
+
+def run(hyper_parameter_map):
+    framework = hyper_parameter_map['framework']
+    if framework is 'keras':
+        import p1b1_baseline_keras2
+        pkg = p1b1_baseline_keras2
+    elif framework is 'mxnet':
+        import p1b1_baseline_mxnet
+        pkg = p1b1_baseline_mxnet
+    elif framework is 'neon':
+        import p1b1_baseline_neon
+        pkg = p1b1_baseline_neon
+    else:
+        raise ValueError("Invalid framework: {}".format(framework))
+
+    # params is python dictionary
+    sys.argv = ['fail here', '--epochs', '54321']
+    params = pkg.initialize_parameters()
+    format_params(hyper_parameter_map)
+
+    for k,v in hyper_parameter_map.items():
+        #if not k in params:
+        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
+        params[k] = v
+
+    print(params)
+    write_params(params, hyper_parameter_map)
+    history = pkg.run(params)
+
+    if framework is 'keras':
+        # works around this error:
+        # https://github.com/tensorflow/tensorflow/issues/3388
+        try:
+            from keras import backend as K
+            K.clear_session()
+        except AttributeError:      # theano does not have this function
+            pass
+
+    # use the last validation_loss as the value to minimize
+    val_loss = history.history['val_loss']
+    return val_loss[-1]
+
+def write_output(result, instance_directory):
+    with open('{}/result.txt'.format(instance_directory), 'w') as f_out:
+        f_out.write("{}\n".format(result))
+
+def init(param_file, instance_directory):
+    with open(param_file) as f_in:
+        hyper_parameter_map = json.load(f_in)
+
+    hyper_parameter_map['framework'] = 'keras'
+    hyper_parameter_map['save'] = '{}/output'.format(instance_directory)
+    hyper_parameter_map['instance_directory'] = instance_directory
+    
+    return hyper_parameter_map
+
+if __name__ == '__main__':
+    print('p1b1_runner main ' + str(argv))
+    param_file = sys.argv[1]
+    instance_directory = sys.argv[2]
+    hyper_parameter_map = init(param_file, instance_directory)
+    # clear sys.argv so that argparse doesn't object
+    sys.argv = ['p1b1_runner']
+    result = run(hyper_parameter_map)
+    write_output(result, instance_directory)

workflows/p1b1_random/python/test/run_test_p1b1.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env bash
+
+P1B1_DIR=../../../../Benchmarks/Pilot1/P1B1
+export PYTHONPATH="$PWD/..:$P1B1_DIR"
+echo $PYTHONPATH
+
+python test_p1b1.py

workflows/p1b1_random/python/test/test_p1b1.py

@@ -0,0 +1,14 @@
+import p1b1_runner
+
+def main():
+
+    hyper_parameter_map = {'epochs' : 1}
+    hyper_parameter_map['batch_size'] = 40
+    hyper_parameter_map['dense'] = [1900, 500] 
+    hyper_parameter_map['framework'] = 'keras'
+    hyper_parameter_map['save'] = './p1bl1_output'
+
+    validation_loss = p1b1_runner.run(hyper_parameter_map)
+    print("Validation Loss: ", validation_loss)
+if __name__ == '__main__':
+    main()

workflows/p1b1_random/swift/computeStats.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python $APP_HOME/../python/computeStats.py $1

workflows/p1b1_random/swift/determineParameters.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo $APP_HOME
+python $APP_HOME/../python/determineParameters.py $1 $2