Added TC1 to Release01

Author: jmohdyusof

Pilot1/TC1/tc1.py

@@ -0,0 +1,72 @@
+from __future__ import print_function
+
+import os
+import sys
+import gzip
+import logging
+
+file_path = os.path.dirname(os.path.realpath(__file__))
+lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
+sys.path.append(lib_path2)
+
+import candle_keras as candle
+
+logger = logging.getLogger(__name__)
+
+additional_definitions = [
+   {'name':'pool',
+       'nargs':'+',
+       'type': int,
+       'help':'network structure of shared layer'},
+]
+
+required = [
+    'data_url',
+    'train_data',
+    'test_data',
+    'model_name',
+    'conv',
+    'dense',
+    'activation',
+    'out_act',
+    'loss',
+    'optimizer',
+    'feature_subsample',
+    'metrics',
+    'epochs',
+    'batch_size',
+    'drop',
+    'classes',
+    'pool',
+    'save'
+]
+
+
+class BenchmarkTC1(candle.Benchmark):
+
+    def set_locals(self):
+        """Functionality to set variables specific for the benchmark
+        - required: set of required parameters for the benchmark.
+        - additional_definitions: list of dictionaries describing the additional parameters for the
+        benchmark.
+        """
+
+        if required is not None:
+            self.required = set(required)
+        if additional_definitions is not None:
+            self.additional_definitions = additional_definitions
+
+
+def load_data(params):
+
+    train_path = candle.fetch_file(params['data_url'] + params['train_data'], 'Pilot1')
+    test_path = candle.fetch_file(params['data_url'] + params['test_data'], 'Pilot1')
+    
+    if params['feature_subsample'] > 0:
+        usecols = list(range(params['feature_subsample']))
+    else:
+        usecols = None
+
+
+    return candle.load_Xy_data_noheader(train_path, test_path, params['classes'], usecols,
+                                        scaling='maxabs',dtype=params['datatype'])

Pilot1/TC1/tc1_baseline_keras2.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import pandas as pd
 import numpy as np
 import os
@@ -11,7 +13,7 @@
 
 from keras import backend as K
 
-from keras.layers import Input, Dense, Dropout, Activation, Conv1D, MaxPooling1D, Flatten, LocallyConnected1D
+from keras.layers import Input, Dense, Dropout, Activation, Conv1D, MaxPooling1D, Flatten
 from keras.optimizers import SGD, Adam, RMSprop
 from keras.models import Sequential, Model, model_from_json, model_from_yaml
 from keras.utils import np_utils
@@ -21,135 +23,29 @@
 from sklearn.preprocessing import StandardScaler, MinMaxScaler, MaxAbsScaler
 
 file_path = os.path.dirname(os.path.realpath(__file__))
-lib_path = os.path.abspath(os.path.join(file_path, '..', 'common'))
-sys.path.append(lib_path)
 lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
 sys.path.append(lib_path2)
 
-import data_utils
-import p1_common
-
-#EPOCH = 400
-#BATCH = 20
-#CLASSES = 2
-
-#PL = 60484   # 1 + 60483 these are the width of the RNAseq datasets
-#P     = 60483   # 60483
-#DR    = 0.1      # Dropout rate
-
-def common_parser(parser):
-
-    parser.add_argument("--config_file", dest='config_file', type=str,
-                        default=os.path.join(file_path, 'tc1_default_model.txt'),
-                        help="specify model configuration file")
-
-    # Parse has been split between arguments that are common with the default neon parser
-    # and all the other options
-    parser = p1_common.get_default_neon_parse(parser)
-    parser = p1_common.get_p1_common_parser(parser)
-
-    return parser
-
-def get_tc1_parser():
-
-	parser = argparse.ArgumentParser(prog='tc1_baseline', formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-                                     description='Train Autoencoder - Pilot 1 Benchmark 1')
+import tc1 as bmk
+import candle_keras as candle
 
-	return common_parser(parser)
-
-def read_config_file(file):
-    config=configparser.ConfigParser()
-    config.read(file)
-    section=config.sections()
-    fileParams={}
-
-    fileParams['data_url']=eval(config.get(section[0],'data_url'))
-    fileParams['train_data']=eval(config.get(section[0],'train_data'))
-    fileParams['test_data']=eval(config.get(section[0],'test_data'))
-    fileParams['model_name']=eval(config.get(section[0],'model_name'))
-    fileParams['conv']=eval(config.get(section[0],'conv'))
-    fileParams['dense']=eval(config.get(section[0],'dense'))
-    fileParams['activation']=eval(config.get(section[0],'activation'))
-    fileParams['out_act']=eval(config.get(section[0],'out_act'))
-    fileParams['loss']=eval(config.get(section[0],'loss'))
-    fileParams['optimizer']=eval(config.get(section[0],'optimizer'))
-    fileParams['feature_subsample']=eval(config.get(section[0],'feature_subsample'))
-    fileParams['metrics']=eval(config.get(section[0],'metrics'))
-    fileParams['epochs']=eval(config.get(section[0],'epochs'))
-    fileParams['batch_size']=eval(config.get(section[0],'batch_size'))
-    fileParams['drop']=eval(config.get(section[0],'drop'))
-    fileParams['classes']=eval(config.get(section[0],'classes'))
-    fileParams['pool']=eval(config.get(section[0],'pool'))
-    fileParams['save']=eval(config.get(section[0], 'save'))
-
-    return fileParams
 
 def initialize_parameters():
-    # Get command-line parameters
-    parser = get_tc1_parser()
-    args = parser.parse_args()
-    #print('Args:', args)
-    # Get parameters from configuration file
-    fileParameters = read_config_file(args.config_file)
-    #print ('Params:', fileParameters)
-    # Consolidate parameter set. Command-line parameters overwrite file configuration
-    gParameters = p1_common.args_overwrite_config(args, fileParameters)
-    return gParameters
-
-
-def load_data(train_path, test_path, gParameters):
-
-    print('Loading data...')
-    if gParameters['feature_subsample'] > 0:
-        usecols = list(range(gParameters['feature_subsample']))
-    else:
-        usecols = None
-    df_train = (pd.read_csv(train_path, header=None, usecols=usecols).values).astype('float32')
-    df_test = (pd.read_csv(test_path, header=None, usecols=usecols).values).astype('float32')
-    print('done')
-
-    print('df_train shape:', df_train.shape)
-    print('df_test shape:', df_test.shape)
 
-    seqlen = df_train.shape[1]
+    # Build benchmark object
+    tc1Bmk = bmk.BenchmarkTC1(file_path, 'tc1_default_model.txt', 'keras',
+    prog='tc1_baseline', desc='Multi-task (DNN) for data extraction from clinical reports - Pilot 3 Benchmark 1')
 
-    df_y_train = df_train[:,0].astype('int')
-    df_y_test = df_test[:,0].astype('int')
+    # Initialize parameters
+    gParameters = candle.initialize_parameters(tc1Bmk)
+    #benchmark.logger.info('Params: {}'.format(gParameters))
 
-    Y_train = np_utils.to_categorical(df_y_train,gParameters['classes'])
-    Y_test = np_utils.to_categorical(df_y_test,gParameters['classes'])
-
-    df_x_train = df_train[:, 1:seqlen].astype(np.float32)
-    df_x_test = df_test[:, 1:seqlen].astype(np.float32)
-
-#        X_train = df_x_train.as_matrix()
-#        X_test = df_x_test.as_matrix()
-
-    X_train = df_x_train
-    X_test = df_x_test
-
-    scaler = MaxAbsScaler()
-    mat = np.concatenate((X_train, X_test), axis=0)
-    mat = scaler.fit_transform(mat)
-
-    X_train = mat[:X_train.shape[0], :]
-    X_test = mat[X_train.shape[0]:, :]
-
-    return X_train, Y_train, X_test, Y_test
+    return gParameters
 
 
 def run(gParameters):
 
-    print ('Params:', gParameters)
-
-    file_train = gParameters['train_data']
-    file_test = gParameters['test_data']
-    url = gParameters['data_url']
-
-    train_file = data_utils.get_file(file_train, url+file_train, cache_subdir='Pilot1')
-    test_file = data_utils.get_file(file_test, url+file_test, cache_subdir='Pilot1')
-
-    X_train, Y_train, X_test, Y_test = load_data(train_file, test_file, gParameters)
+    X_train, Y_train, X_test, Y_test = bmk.load_data(gParameters)
 
     print('X_train shape:', X_train.shape)
     print('X_test shape:', X_test.shape)
@@ -169,6 +65,7 @@ def run(gParameters):
 
     model = Sequential()
     dense_first = True
+
     layer_list = list(range(0, len(gParameters['conv']), 3))
     for l, i in enumerate(layer_list):
         filters = gParameters['conv'][i]
@@ -212,26 +109,8 @@ def run(gParameters):
         model.add(Flatten())
 
     model.add(Dense(gParameters['classes']))
-
     model.add(Activation(gParameters['out_act']))
 
-#Reference case
-#model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1)))
-#model.add(Activation('relu'))
-#model.add(MaxPooling1D(pool_size=1))
-#model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid'))
-#model.add(Activation('relu'))
-#model.add(MaxPooling1D(pool_size=10))
-#model.add(Flatten())
-#model.add(Dense(200))
-#model.add(Activation('relu'))
-#model.add(Dropout(0.1))
-#model.add(Dense(20))
-#model.add(Activation('relu'))
-#model.add(Dropout(0.1))
-#model.add(Dense(CLASSES))
-#model.add(Activation('softmax'))
-
     model.summary()
 
     model.compile(loss=gParameters['loss'],
@@ -241,8 +120,8 @@ def run(gParameters):
     output_dir = gParameters['save']
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
-# set up a bunch of callbacks to do work during model training..
-
+    
+    # set up callbacks to do work during model training..
     model_name = gParameters['model_name']
     path = '{}/{}.autosave.model.h5'.format(output_dir, model_name)
     checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True)
@@ -324,6 +203,7 @@ def run(gParameters):
 
     return history
 
+
 def main():
 
     gParameters = initialize_parameters()
@@ -335,3 +215,4 @@ def main():
         K.clear_session()
     except AttributeError:      # theano does not have this function
         pass
+