o Add a general workflow that is capable of running any benchmark and…

workflows/rnd_or_grid/README.md

@@ -0,0 +1,13 @@
+# Simple grid or random parameter sweep with Swift for all the benchmarks, using command line. D type , which runs a parameter sweep. It calls command-line programs as follows:
+- determineParameters.{sh,py}: Reads data/**settings.json** for sweep parameters, and return as a string for use by Swift program (sweep-parameters.txt)
+- evaluateOne.{sh,py}: Runs a single experiment. (Calls the specified benchmark).
+- computeStats.{sh,py}: Ingests data from all of the experiments and computes simple stats.
+
+Usage: ./run <run directory> <benchmark name> <search type> ./run ex3_p1b1_grid p1b1 grid
+
+Notes:
+**settings.json**: sweep parameters variation
+1. json file must be present in the data folder and named as: <benchmark name>_settings.json, samples files are available and must be modified as per needs.
+2. Run directory will be created in the experiments folder
+3. New variables can be introduced in the determineParameters.py and evaluateOne.py. 
+4. Variations of parameters must be specified in data/*.json files

workflows/rnd_or_grid/data/nt3_settings.json

@@ -0,0 +1,12 @@
+{
+    "parameters":
+          { 
+          	"epochs": [1, 2 ],
+            "batch_size": [10, 20],
+            "classes": [2, 3]
+          },
+    "samples":
+    	  {
+    	  	"num": [2]
+    	  }
+}

workflows/rnd_or_grid/data/p1b1_settings.json

@@ -0,0 +1,13 @@
+{
+    "parameters":
+          { 
+            "epochs": [1, 2, 8],
+            "batch_size": [20, 40],
+            "N1": [1000, 2000], 
+            "NE": [500, 600]
+          },
+    "samples":
+          {
+            "num": [2]
+          }
+}

workflows/rnd_or_grid/data/p1b3_settings.json

@@ -0,0 +1,14 @@
+{
+    "parameters":
+          { 
+          	"epochs": [1, 2],
+            "batch_size": [50, 60],
+            "test_cell_split": [0.15, 0.25], 
+            "drop": [0.1, 0.15]
+          },
+    "samples":
+          {
+            "num": [2]
+          }
+
+}

workflows/rnd_or_grid/data/p2b1_settings.json

@@ -0,0 +1,13 @@
+{
+    "parameters":
+          { 
+            "epochs": [1, 2],
+            "batch_size": [32, 40],
+            "molecular_epochs": [1, 3], 
+            "weight_decay": [0.0005, 0.0006]
+          },
+    "samples":
+          {
+            "num": [2]
+          }
+}

workflows/rnd_or_grid/data/p3b1_settings.json

@@ -0,0 +1,12 @@
+{
+    "parameters":
+          { 
+          	"epochs": [1 , 2 ],
+            "batch_size": [20, 40]
+          },
+    "samples":
+          {
+            "num": [2]
+          }
+}
+

workflows/rnd_or_grid/python/computeStats.py

@@ -0,0 +1,40 @@
+import sys
+from collections import defaultdict
+import json, os
+
+def extractVals(A):
+    B = defaultdict(dict)
+    A1 = A.split()
+    for n, val in zip(A1[0::2], A1[1::2]):
+        B[n] = float(val)
+    return(B)
+
+def computeStats(swiftArrayAsString):
+    A = extractVals(swiftArrayAsString)
+    vals = []
+    for a in A:
+        vals += [A[a]]
+    print('%d values, with min=%f, max=%f, avg=%f\n'%(len(vals),min(vals),max(vals),sum(vals)/float(len(vals))))
+
+    filename = os.environ['TURBINE_OUTPUT']+ "/final_stats.txt"
+    # writing the val loss to the output file
+    with open(filename, 'w') as the_file:
+        the_file.write('%d values, with min=%f, max=%f, avg=%f\n'%(len(vals),min(vals),max(vals),sum(vals)/float(len(vals))))
+
+
+
+if (len(sys.argv) < 2):
+	print('requires arg=dataFilename')
+	sys.exit(1)
+
+dataFilename = sys.argv[1]
+
+try:
+    with open(dataFilename, 'r') as the_file:
+        data = the_file.read()
+except IOError as e:
+    print("Could not open: %s" % dataFilename)
+    print("PWD is: '%s'" % os.getcwd())
+
+computeStats(data)
+

workflows/rnd_or_grid/python/determineParameters.py

@@ -0,0 +1,154 @@
+import sys, json, os
+import random 
+import itertools
+
+# ===== Definitions =========================================================
+def expand(Vs, fr, to, soFar):
+    soFarNew = []
+    for s in soFar:
+        if (Vs[fr] == None):
+             print ("ERROR: The order of json inputs and values must be preserved")
+             sys.exit(1)
+        for v in Vs[fr]:
+             if s == '':
+                 soFarNew += [str(v)]
+             else:
+                 soFarNew += [s+','+str(v)]
+    if fr==to:
+        return(soFarNew)
+    else:
+        return expand(Vs, fr+1, to, soFarNew)
+
+def generate_random(values, n_samples, benchmarkName):
+    # select '#samples' random numbers between the range provided in settings.json file
+    result = ""
+    param_listed = []
+    for s in range(n_samples):
+        if(benchmarkName=="p1b1"):
+            # values = {1:epochs, 2: batch_size, 3: N1, 4: NE}
+            t_epoch= random.randint(values[1][0], values[1][1])
+            t_batch_size= random.randint(values[2][0], values[2][1])
+            t_N1= random.randint(values[3][0], values[3][1])
+            t_NE= random.randint(values[4][0], values[4][1])
+            result+=str(t_epoch) + ',' + str(t_batch_size) + ',' + str(t_N1) + ',' + str(t_NE) 
+        elif(benchmarkName=="p1b3"):
+            # values = {1:epochs, 2: batch_size, 3: test_cell_split, 4: drop}
+            t_epoch= random.randint(values[1][0], values[1][1])
+            t_batch_size= random.randint(values[2][0], values[2][1])
+            t_tcs= random.uniform(values[3][0], values[3][1])
+            t_drop= random.uniform(values[4][0], values[4][1])
+            result+=str(t_epoch) + ',' + str(t_batch_size) + ',' + str(t_tcs) + ',' + str(t_drop) 
+        elif(benchmarkName=="nt3"):
+            # values = {1:epochs, 2: batch_size, 3: classes}
+            t_epoch= random.randint(values[1][0], values[1][1])
+            t_batch_size= random.randint(values[2][0], values[2][1])
+            t_classes= random.randint(values[3][0], values[3][1])
+            result+=str(t_epoch) + ',' + str(t_batch_size) + ',' + str(t_classes)  
+        elif(benchmarkName=="p2b1"):
+            # values = {1:epochs, 2: batch_size, 3: molecular_epochs, 4: weight_decay}
+            t_epoch= random.randint(values[1][0], values[1][1])
+            t_batch_size= random.randint(values[2][0], values[2][1])
+            t_me= random.randint(values[3][0], values[3][1])
+            t_wd= random.uniform(values[4][0], values[4][1])
+            result+=str(t_epoch) + ',' + str(t_batch_size) + ',' + str(t_me) + ',' + str(t_wd) 
+        elif(benchmarkName=="p3b1"):
+            # values = {1:epochs, 2: batch_size}//, 3: learning_rate, 4: n_fold}
+            t_epoch= random.randint(values[1][0], values[1][1])
+            t_batch_size= random.randint(values[2][0], values[2][1])
+            result+=str(t_epoch) + ',' + str(t_batch_size)
+        else:
+            print('ERROR: Tried all possible benchmarks, Invalid benchmark name or json file')
+            sys.exit(1)
+        # Populate the result string for writing sweep-parameters file
+        param_listed += [str(result)]
+        result=""
+    return (param_listed)
+
+# ===== Main program ========================================================
+if (len(sys.argv) < 3):
+    print('requires arg1=settingsFilename and arg2=paramsFilename')
+    sys.exit(1)
+
+settingsFilename = sys.argv[1]
+paramsFilename   = sys.argv[2]
+benchmarkName    = sys.argv[3]
+searchType       = sys.argv[4]
+
+## Read in the variables from json file
+#Trying to open the settings file
+print("Reading settings: %s" % settingsFilename)
+try:
+    with open(settingsFilename) as fp:
+        settings = json.load(fp)
+except IOError as e:
+    print("Could not open: %s" % settingsFilename)
+    print("PWD is: '%s'" % os.getcwd())
+    sys.exit(1)
+
+# Register new variables for any benchmark here
+#Common variables
+epochs = settings.get('parameters').get('epochs')
+batch_size = settings.get('parameters').get('batch_size')
+# P1B1
+N1 = settings.get('parameters').get('N1')
+NE = settings.get('parameters').get('NE') 
+#NT3
+classes = settings.get('parameters').get('classes')   
+#P2B1
+molecular_epochs = settings.get('parameters').get('molecular_epochs') 
+weight_decay = settings.get('parameters').get('weight_decay') 
+#P3B1
+# learning_rate = settings.get('parameters').get('learning_rate') 
+# n_fold = settings.get('parameters').get('n_fold') 
+#P1B3
+test_cell_split = settings.get('parameters').get('test_cell_split') 
+drop = settings.get('parameters').get('drop') 
+
+# For random scheme determine number of samples
+samples = settings.get('samples', {}).get('num', None)
+
+## Done reading from file 
+
+# Make values for computing grid sweep parameters
+values = {}
+if(benchmarkName=="p1b1"):
+    values = {1:epochs, 2: batch_size, 3: N1, 4: NE}
+elif(benchmarkName=="p1b3"):
+    values = {1:epochs, 2: batch_size, 3: test_cell_split, 4: drop}
+elif(benchmarkName=="nt3"):
+    values = {1:epochs, 2: batch_size, 3: classes}
+elif(benchmarkName=="p2b1"):
+    values = {1:epochs, 2: batch_size, 3: molecular_epochs, 4: weight_decay}
+elif(benchmarkName=="p3b1"):
+    values = {1:epochs, 2: batch_size}
+else:
+    print('ERROR: Tried all possible benchmarks, Invalid benchmark name or json file')
+    sys.exit(1)
+
+# this (result) is : seperated string with all params
+result = {}
+# Determine parameter space based of search type
+if(searchType == "grid"):
+    results = expand(values, 1, len(values), [''])
+elif(searchType =="random"):
+    if(samples == None):
+        print ("ERROR: Provide number of samples in json file")
+        sys.exit(1)
+    # result, results = generate_random(values, samples, benchmarkName) 
+    results = generate_random(values, samples[0], benchmarkName) 
+else:
+    print ("ERROR: Invalid search type, specify either - grid or random")
+    sys.exit(1)
+
+counter=0
+for a, b in itertools.combinations(results, 2):
+    if(a == b):
+        print ("Warning: skipping -identical parameters found", counter)
+        results.remove(a)
+
+#These are final : seperated parameters for evaluation
+result = ':'.join(results)
+
+with open(paramsFilename, 'w') as the_file:
+    the_file.write(result)
+

workflows/rnd_or_grid/python/evaluateOne.py

@@ -0,0 +1,90 @@
+import sys
+import json, os
+import socket
+
+
+if (len(sys.argv) < 3):
+	print('requires arg1=param and arg2=filename')
+	sys.exit(1)
+
+parameterString = sys.argv[1]
+filename        = sys.argv[2]
+benchmarkName   = sys.argv[3]
+
+integs = [float(x) for x in parameterString.split(',')]
+
+if (benchmarkName == "p1b1"):
+	import p1b1_runner
+	hyper_parameter_map = {'epochs' : int(integs[0])}
+	hyper_parameter_map['framework'] = 'keras'
+	hyper_parameter_map['batch_size'] = int(integs[1])
+	hyper_parameter_map['dense'] = [int(integs[2]), int(integs[3])] 
+	hyper_parameter_map['run_id'] = parameterString
+	# hyper_parameter_map['instance_directory'] = os.environ['TURBINE_OUTPUT'] 
+	hyper_parameter_map['save'] = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid())
+	sys.argv = ['p1b1_runner']
+	val_loss = p1b1_runner.run(hyper_parameter_map)
+elif (benchmarkName == "p1b3"):
+	import p1b3_runner
+	hyper_parameter_map = {'epochs' : int(integs[0])}
+	hyper_parameter_map['framework'] = 'keras'
+	hyper_parameter_map['batch_size'] = int(integs[1])
+	hyper_parameter_map['test_cell_split'] = int(integs[2])
+	hyper_parameter_map['drop'] = int(integs[3])
+	hyper_parameter_map['run_id'] = parameterString
+	# hyper_parameter_map['instance_directory'] = os.environ['TURBINE_OUTPUT'] 
+	hyper_parameter_map['save'] = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid())
+	sys.argv = ['p1b3_runner']
+	val_loss = p1b3_runner.run(hyper_parameter_map)
+elif (benchmarkName == "p2b1"):
+	import p2b1_runner
+	hyper_parameter_map = {'epochs' : int(integs[0])}
+	hyper_parameter_map['framework'] = 'keras'
+	hyper_parameter_map['batch_size'] = int(integs[1])
+	hyper_parameter_map['molecular_epochs'] = int(integs[2])
+	hyper_parameter_map['weight_decay'] = integs[3]
+	hyper_parameter_map['run_id'] = parameterString
+	hyper_parameter_map['save'] = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid())
+	sys.argv = ['p2b1_runner']
+	val_loss = p2b1_runner.run(hyper_parameter_map)
+elif (benchmarkName == "nt3"):
+	import nt3_tc1_runner
+	hyper_parameter_map = {'epochs' : int(integs[0])}
+	hyper_parameter_map['framework'] = 'keras'
+	hyper_parameter_map['batch_size'] = int(integs[1])
+	hyper_parameter_map['classes'] = int(integs[2])
+	hyper_parameter_map['model_name'] = 'nt3'	
+	hyper_parameter_map['save'] = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid())
+	sys.argv = ['nt3_runner']
+	val_loss = nt3_tc1_runner.run(hyper_parameter_map)
+elif (benchmarkName == "p3b1"):
+	import p3b1_runner
+	hyper_parameter_map = {'epochs' : int(integs[0])}
+	hyper_parameter_map['framework'] = 'keras'
+	hyper_parameter_map['batch_size'] = int(integs[1])
+	hyper_parameter_map['run_id'] = parameterString
+	# hyper_parameter_map['instance_directory'] = os.environ['TURBINE_OUTPUT'] 
+	hyper_parameter_map['save'] = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid())
+	sys.argv = ['p3b1_runner']
+	val_loss = p3b1_runner.run(hyper_parameter_map)
+
+# print (parameterString)
+# print ("filename is " + filename)
+# print (str(os.getpid()))
+print (val_loss)
+
+# sfn = os.environ['TURBINE_OUTPUT']+ "/output-"+str(os.getpid()) + "/procname-" + parameterString
+# with open(sfn, 'w') as sfile:
+#     sfile.write(socket.getfqdn())
+#     proc_id = "-"+ str(os.getpid())
+#     sfile.write(proc_id)
+
+# works around this error:
+# https://github.com/tensorflow/tensorflow/issues/3388
+from keras import backend as K
+K.clear_session()
+
+# writing the val loss to the output file (result-*)
+with open(filename, 'w') as the_file:
+    the_file.write(repr(val_loss))
+