Merge branch 'master' of github.com:ECP-CANDLE/Supervisor

Author: ncollier

workflows/p1b1_mlrMBO/R/mlrMBO_utils.R

@@ -1,4 +1,6 @@
 # Utility code to transform elements to strings and vice versa
+library(jsonlite)
+
 elements_to_string <- function(x){
   paste0(x,collapse = ",")
 }
@@ -11,10 +13,9 @@ list_to_string <- function(x){
   toString(x)
 }
 
-elements_of_lists_to_string <- function(x){
-  paste0(sapply(x,list_to_string),collapse = ';')
+elements_of_lists_to_json <- function(x){
+  paste0(sapply(x,toJSON,auto_unbox = T),collapse = ';')
 }
-
 string_to_list_of_vectors <- function(x){
   lapply(unlist(strsplit(x,";")),function(y) as.numeric(unlist(strsplit(y,","))))
 }
@@ -31,3 +32,26 @@ result_with_extras_if_exist <- function(res,time_value){
   lapply(res, function(x) append_extras_if_exist(c(list(y=x[1]),
                                                    list(time=time_value)),x))
 }
+
+strsubst <-
+function(template, map, verbose=getOption("verbose")) {
+  pat <- "\\$\\([^\\)]+\\)"
+  res <- template
+  map[["$"]] <- "$"
+  m <- gregexpr(pat, template)
+  idx <- which(sapply(m, function(x) x[[1]]!=-1)) # faster than 1:length(template)?
+  for (i in idx) {
+    line <- template[[i]]
+    if(verbose) cat("input: |", template[[i]], "|\n")
+    starts <- m[[i]]
+    ml <- attr(m[[i]], "match.length")
+    sym <- substring(line, starts+2, starts+ml-2)
+    repl <- map[sym]
+    idx1 <- is.null(repl)
+    repl[idx1] <- sym[idx1]
+    norepl <- substring(line, c(1, starts+ml), c(starts-1, nchar(line)))
+    res[[i]] <- paste(norepl, c(repl, ""), sep="", collapse="") # more elegant?
+    if (verbose) cat("output: |", res[[i]], "|\n")
+  }
+  return(res)
+}

workflows/p1b1_mlrMBO/swift/workflow.sh

@@ -27,7 +27,7 @@ export PROCS=4
 # be ignored if MACHINE flag (see below) is not set
 export QUEUE=batch
 export WALLTIME=00:10:00
-export PPN=16
+export PPN=1
 export TURBINE_JOBNAME="${EXPID}_job"
 
 # if R cannot be found, then these will need to be

workflows/p1b3_mlrMBO/python/stage_data.py

@@ -0,0 +1,2 @@
+import p1b3
+p1b3.stage_data()

workflows/p1b3_mlrMBO/scripts/get_data.sh

@@ -0,0 +1,10 @@
+
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_cellline_expressions.tsv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_cellline_mirna.tsv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_cellline_proteome.tsv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_cellline_kinome.tsv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_drug_descriptors.tsv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_drug_latent.csv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_dose_response.csv
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_test_celllines.txt
+wget http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/P1B3_test_drugs.txt

workflows/p1b3_mlrMBO/scripts/stage_data.sh

@@ -0,0 +1,69 @@
+#!/bin/bash
+
+set -eu
+
+# Check for an optional timeout threshold in seconds. If the duration of the
+# model run as executed below, takes longer that this threshhold
+# then the run will be aborted. Note that the "timeout" command
+# must be supported by executing OS.
+
+# The timeout argument is optional. By default the "run_model" swift
+# app fuction sends 3 arguments, and no timeout value is set. If there
+# is a 4th (the TIMEOUT_ARG_INDEX) argument, we use that as the timeout value.
+
+# !!! IF YOU CHANGE THE NUMBER OF ARGUMENTS PASSED TO THIS SCRIPT, YOU MUST
+# CHANGE THE TIMEOUT_ARG_INDEX !!!
+TIMEOUT_ARG_INDEX=4
+TIMEOUT=""
+if [[ $# ==  $TIMEOUT_ARG_INDEX ]]
+then
+	TIMEOUT=${!TIMEOUT_ARG_INDEX}
+fi
+
+TIMEOUT_CMD=""
+if [ -n "$TIMEOUT" ]; then
+  TIMEOUT_CMD="timeout $TIMEOUT"
+fi
+
+# Set param_line from the first argument to this script
+# param_line is the string containing the model parameters for a run.
+param_file=$1
+
+# Set emews_root to the root directory of the project (i.e. the directory
+# that contains the scripts, swift, etc. directories and files)
+emews_root=$2
+
+# Each model run, runs in its own "instance" directory
+# Set instance_directory to that and cd into it.
+instance_directory=$3
+cd $instance_directory
+
+# TODO: Define the command to run the model
+#VERSION="$(<$emews_root/../Release/version.txt)"
+#APP=$emews_root/../Release/transmission_model-$VERSION
+#PROPS_FILE=$emews_root/../config/model.props
+# TODO configure python correctly
+# export PYTHONPATH=$emews_root/python:$benchmark_path
+PYTHON="/home/pbalapra/anaconda2/envs/idp/bin/python"
+export LD_LIBRARY_PATH="/home/pbalapra/anaconda2/envs/idp/lib"
+export PATH="/home/pbalapra/anaconda2/envs/idp/bin:$PATH"
+export PYTHONHOME="/home/pbalapra/anaconda2/envs/idp"
+export PYTHONPATH="/home/pbalapra/anaconda2/envs/idp/lib/python2.7:/home/ncollier/repos/Benchmarks/Pilot1/P1B3:/home/pbalapra/anaconda2/envs/idp/lib/python2.7/site-packages"
+MODEL_CMD="python $emews_root/python/stage_data.py"
+
+# Turn bash error checking off. This is
+# required to properly handle the model execution return value
+# the optional timeout.
+set +e
+echo $MODEL_CMD
+$TIMEOUT_CMD $MODEL_CMD
+# $? is the exit status of the most recently executed command (i.e the
+# line above)
+RES=$?
+if [ "$RES" -ne 0 ]; then
+	if [ "$RES" == 124 ]; then
+    echo "---> Timeout error in $MODEL_CMD"
+  else
+	   echo "---> Error in $MODEL_CMD"
+  fi
+fi

workflows/p1b3_mlrMBO/scripts/theta_run_model.sh

@@ -48,7 +48,7 @@ PYTHON="/home/pbalapra/anaconda2/envs/idp/bin/python"
 export LD_LIBRARY_PATH="/home/pbalapra/anaconda2/envs/idp/lib"
 export PATH="/home/pbalapra/anaconda2/envs/idp/bin:$PATH"
 export PYTHONHOME="/home/pbalapra/anaconda2/envs/idp"
-export PYTHONPATH="/home/pbalapra/anaconda2/envs/idp/lib/python2.7:/home/ncollier/repos/Benchmarks/Pilot1/P1B3"
+export PYTHONPATH="/home/pbalapra/anaconda2/envs/idp/lib/python2.7:/home/ncollier/repos/Benchmarks/Pilot1/P1B3:/home/pbalapra/anaconda2/envs/idp/lib/python2.7/site-packages"
 MODEL_CMD="python $emews_root/python/p1b3_runner.py $param_file $instance_directory"
 
 # Turn bash error checking off. This is