o Adding simple parameter grid search with swift for P1B1

o See README for details

Author: rajeeja

workflows/p1b1_grid/README.md

@@ -0,0 +1,14 @@
+# Simple parameter sweep with Swift, using command line programs
+**run** runs **run-sweep.swift**, which runs a parameter sweep. It calls command-line programs as follows:
+- determineParameters.{sh,py}: Read data/ **settings.json** for sweep parameters, and return as a string for use by Swift program
+- evaluateOne.{sh,py}: Runs a single experiment. (Calls p1b1_baseline).
+- computeStats.{sh,py}: Ingests data from all of the experiments and computes simple stats.
+
+Usage: ./run 
+
+Notes:
+- **settings.json**: sweep parameters. Parameters must be labeled "1", "2", "3", "4", ... 
+1: epochs
+2. batch_size
+3. N1
+4. NE

workflows/p1b1_grid/data/settings.json

@@ -0,0 +1,7 @@
+{
+    "parameters":
+          { 
+          	"1": [2,4],
+            "2": [5, 6, 7]
+          }
+}

workflows/p1b1_grid/python/computeStats.py

@@ -0,0 +1,34 @@
+import sys
+from collections import defaultdict
+import json, os
+
+def extractVals(A):
+    B = defaultdict(dict)
+    A1 = A.split()
+    for n, val in zip(A1[0::2], A1[1::2]):
+        B[n] = float(val)
+    return(B)
+
+def computeStats(swiftArrayAsString):
+    A = extractVals(swiftArrayAsString)
+    vals = []
+    for a in A:
+        vals += [A[a]]
+    print('%d values, with min=%f, max=%f, avg=%f\n'%(len(vals),min(vals),max(vals),sum(vals)/float(len(vals))))
+
+
+if (len(sys.argv) < 2):
+	print('requires arg=dataFilename')
+	sys.exit(1)
+
+dataFilename = sys.argv[1]
+
+try:
+    with open(dataFilename, 'r') as the_file:
+        data = the_file.read()
+except IOError as e:
+    print("Could not open: %s" % dataFilename)
+    print("PWD is: '%s'" % os.getcwd())
+
+computeStats(data)
+

workflows/p1b1_grid/python/determineParameters.py

@@ -0,0 +1,57 @@
+import sys, json, os
+
+# ===== Definitions =========================================================
+
+def loadSettings(settingsFilename):
+    print("Reading settings: %s" % settingsFilename)
+    try:
+        with open(settingsFilename) as fp:
+            settings = json.load(fp)
+    except IOError as e:
+        print("Could not open: %s" % settingsFilename)
+        print("PWD is: '%s'" % os.getcwd())
+        sys.exit(1)
+    try:
+        params = settings['parameters']
+    except KeyError as e:
+        print("Settings file (%s) does not contain key: %s" % (settingsFilename, str(e)))
+        sys.exit(1)
+    return(params)
+
+def expand(Vs, fr, to, soFar):
+    soFarNew = []
+    for s in soFar:
+        for v in Vs[fr]:
+             if s == '':
+                 soFarNew += [str(v)]
+             else:
+                 soFarNew += [s+','+str(v)]
+    if fr==to:
+        return(soFarNew)
+    else:
+        return expand(Vs, fr+1, to, soFarNew)
+
+# ===== Main program ========================================================
+
+if (len(sys.argv) < 3):
+	print('requires arg1=settingsFilename and arg2=paramsFilename')
+	sys.exit(1)
+
+settingsFilename = sys.argv[1]
+paramsFilename   = sys.argv[2]
+
+params = loadSettings(settingsFilename)
+values = {}
+for i in range(1, len(params)+1):
+    try:
+         As = params[str(i)]
+    except:
+         print('Did not find parameter %i in settings file'%i)
+         sys.exit(1)
+    values[i] = As
+results = expand(values, 1, len(params), [''])
+result = ':'.join(results)
+
+with open(paramsFilename, 'w') as the_file:
+    the_file.write(result)
+

workflows/p1b1_grid/python/evaluateOne.py

@@ -0,0 +1,46 @@
+import sys
+import p1b1_baseline
+import p1b1
+
+if (len(sys.argv) < 3):
+	print('requires arg1=param and arg2=filename')
+	sys.exit(1)
+
+parameterString = sys.argv[1]
+filename        = sys.argv[2]
+
+print (parameterString)
+print ("filename is ", filename)
+
+epochs = int(parameterString[0].strip())
+batch_size = int(parameterString[2].strip())
+print ("Running p1b1 for epochs ", epochs, batch_size)
+
+# N1 = int(parameterString[2].strip())
+# NE = int(parameterString[3].strip())
+
+print("Set the correct paths for test and train file")
+test_path="/home/jain/Benchmarks/Data/Pilot1/P1B1.test.csv"
+train_path="/home/jain/Benchmarks/Data/Pilot1/P1B1.train.csv"
+
+print ("Starting to loading Xtrain and Xtest")
+X_train, X_test = p1b1.load_data(test_path=test_path, train_path=train_path)
+print ("Done loading Xtrain and Xtest")
+
+print ("Running p1b1 for epochs ", epochs)
+encoder, decoder, history = p1b1_baseline.run_p1b1(X_train, X_test, epochs=epochs, batch_size=batch_size)
+print ("Done running p1b1 for epochs ", epochs)
+
+# works around this error:
+# https://github.com/tensorflow/tensorflow/issues/3388
+from keras import backend as K
+K.clear_session()
+
+# use the last validation_loss as the value to minimize
+val_loss = history.history['val_loss']
+r = val_loss[-1]
+
+# writing the val loss to the output file
+with open(filename, 'w') as the_file:
+    the_file.write(repr(r))
+

workflows/p1b1_grid/swift/computeStats.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+python $APP_HOME/../python/computeStats.py $1

workflows/p1b1_grid/swift/determineParameters.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+echo $APP_HOME
+python $APP_HOME/../python/determineParameters.py $1 $2

workflows/p1b1_grid/swift/evaluateOne.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+filename=result-$1.txt
+python $APP_HOME/../python/evaluateOne.py $1 $filename

workflows/p1b1_grid/swift/run

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# Usage: ./run 
+#
+
+#### set this variable to your P1B1 benchmark directory
+P1B1_DIR=/home/jain/Benchmarks/Pilot1/P1B1
+###
+
+THIS=$( cd $( dirname $0 ); /bin/pwd )
+export APP_HOME=$THIS
+
+PROJECT_ROOT=$APP_HOME/..
+
+### set 
+export PYTHONPATH=/usr/local/lib/python2.7/
+###
+export PYTHONPATH=$PYTHONPATH:$PROJECT_ROOT/python:$P1B1_DIR
+
+echo $PYTHONPATH
+
+### set the desired number of processors
+PROCS=2
+###
+
+# remove -l option for removing printing processors ranks
+echo swift-t  -n $PROCS $APP_HOME/run-sweep.swift $*
+swift-t  -l -n $PROCS $APP_HOME/run-sweep.swift $* --settings=$PWD/../data/settings.json

workflows/p1b1_grid/swift/run-sweep.swift

@@ -0,0 +1,50 @@
+import string;
+import files;
+import io;
+import sys;
+
+// ===== Interface definitions for the three programs that we call ======
+app (file f)
+determineParameters(string settingsFilename)
+{
+  (getenv("APP_HOME")+"/determineParameters.sh") settingsFilename f;
+}
+
+app (file f)
+evaluateOne(string params)
+{
+  (getenv("APP_HOME")+"/evaluateOne.sh") params f;
+}
+
+app ()
+computeStats(string resultsFile)
+{
+  (getenv("APP_HOME")+"/computeStats.sh") resultsFile;
+}
+
+
+// ===== The program proper ==============================================
+float results[string];
+
+// Get parameters
+settingsFilename = argv("settings");
+printf(settingsFilename);
+file parametersFile<"parameters.txt"> = determineParameters(settingsFilename);
+parametersString = read(parametersFile);
+parameters = split(parametersString, ":");
+
+// Run experiments in parallel, passing each a different parameter set
+foreach param in parameters
+{
+    file resultFile<"result-%s.txt"%param> = evaluateOne(param);
+    results[param] = string2float(read(resultFile));
+}
+
+// Compute stats of this array of results
+// Write directly to a file with write
+file tmp = write(repr(results));
+
+// Find the name of a file with filename
+//trace("Temporary filename is: " + filename(tmp));
+
+computeStats(filename(tmp));