Add new benchmark P3B3: MT-CNN

Author: Yoon, Hong-Jun

Pilot3/P3B3/keras_mt_shared_cnn.py

@@ -0,0 +1,73 @@
+"""
+Code to export keras architecture/placeholder weights for MT CNN
+Written by Mohammed Alawad
+Date: 10_20_2017
+"""
+import numpy as np
+#np.random.seed(1337)
+from keras.models import Model
+from keras.layers import Dense, Activation, Dropout, Flatten, Input, Embedding
+from keras.layers import merge as Merge
+from keras.layers import GlobalMaxPooling1D, Convolution1D
+#from keras.layers.convolutional import Conv1D
+from keras.layers.merge import Concatenate
+from keras import optimizers
+import keras.backend as K
+from keras.regularizers import l2
+import pickle
+import argparse
+import os
+from keras.initializers import RandomUniform,lecun_uniform
+
+def init_export_network(num_classes,
+                        in_seq_len,
+                        vocab_size,
+                        wv_space,
+                        filter_sizes,
+                        num_filters,
+                        concat_dropout_prob,
+                        emb_l2,
+                        w_l2,
+                        optimizer):
+
+
+    # define network layers ----------------------------------------------------
+    input_shape = tuple([in_seq_len])
+    model_input = Input(shape=input_shape, name= "Input")
+    # embedding lookup
+    emb_lookup = Embedding(vocab_size,
+                           wv_space,
+                           input_length=in_seq_len,
+                           name="embedding",
+                           #embeddings_initializer=RandomUniform,
+                           embeddings_regularizer=l2(emb_l2))(model_input)
+    # convolutional layer and dropout
+    conv_blocks = []
+    for ith_filter,sz in enumerate(filter_sizes):
+        conv = Convolution1D(filters=num_filters[ ith_filter ],
+                             kernel_size=sz,
+                             padding="same",
+                             activation="relu",
+                             strides=1,
+                             # kernel_initializer ='lecun_uniform,
+                             name=str(ith_filter) + "_thfilter")(emb_lookup)
+        conv_blocks.append(GlobalMaxPooling1D()(conv))
+    concat = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
+    concat_drop = Dropout(concat_dropout_prob)(concat)
+
+    # different dense layer per tasks
+    FC_models = []
+    for i in range(len(num_classes)):
+        outlayer = Dense(num_classes[i], name= "Dense"+str(i), activation='softmax')( concat_drop )#, kernel_regularizer=l2(0.01))( concat_drop )
+        FC_models.append(outlayer)
+
+
+    # the multitsk model
+    model = Model(inputs=model_input, outputs = FC_models)
+    model.compile( loss= "sparse_categorical_crossentropy", optimizer= optimizer, metrics=[ "acc" ] )
+
+    return model
+
+if __name__ == '__main__':
+    main()
+

Pilot3/P3B3/p3b3.py

@@ -0,0 +1,113 @@
+from __future__ import absolute_import
+from __future__ import print_function
+import os
+import sys
+import argparse
+try:
+    import configparser
+except ImportError:
+    import ConfigParser as configparser
+
+file_path = os.path.dirname(os.path.realpath(__file__))
+lib_path = os.path.abspath(os.path.join(file_path, '..', 'common'))
+sys.path.append(lib_path)
+lib_path2 = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
+sys.path.append(lib_path2)
+
+import p3_common
+
+def common_parser(parser):
+
+    parser.add_argument("--config_file", dest='config_file', type=str,
+                        default=os.path.join(file_path, 'p3b3_default_model.txt'),
+                        help="specify model configuration file")
+
+    # Parse has been split between arguments that are common with the default neon parser
+    # and all the other options
+    parser = p3_common.get_default_neon_parse(parser)
+    parser = p3_common.get_p3_common_parser(parser)
+
+    # Arguments that are applicable just to p3b1
+    parser = p3b3_parser(parser)
+
+    return parser
+
+def p3b3_parser(parser):
+    ### Hyperparameters and model save path
+
+    # these are leftover from other models but don't conflict so leave for now
+    parser.add_argument("--train", action="store_true",dest="train_bool",default=True,help="Invoke training")
+    parser.add_argument("--evaluate", action="store_true",dest="eval_bool",default=False,help="Use model for inference")
+    parser.add_argument("--home-dir",help="Home Directory",dest="home_dir",type=str,default='.')
+    parser.add_argument("--save-dir",help="Save Directory",dest="save_path",type=str,default=None)
+    parser.add_argument("--config-file",help="Config File",dest="config_file",type=str,default=os.path.join(file_path, 'p3b3_default_model.txt'))
+    parser.add_argument("--memo",help="Memo",dest="base_memo",type=str,default=None)
+    parser.add_argument("--seed", action="store_true",dest="seed",default=False,help="Random Seed")
+    parser.add_argument("--case",help="[Full, Center, CenterZ]",dest="case",type=str,default='CenterZ')
+    parser.add_argument("--fig", action="store_true",dest="fig_bool",default=False,help="Generate Prediction Figure")
+
+    # MTL_run params start here
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        default= True,
+                        help="increase output verbosity")
+
+    parser.add_argument("--dropout", action="store",
+                        default=argparse.SUPPRESS, # DROPOUT, type=float,
+                        help="ratio of dropout used in fully connected layers")
+    parser.add_argument("--learning_rate", action='store',
+                        default=argparse.SUPPRESS, #  LEARNING_RATE, type=float,
+                        help='learning rate')
+
+    parser.add_argument("--train_features", action="store",
+                        default='data/train_X.npy',
+                        help='training feature data filenames')
+    parser.add_argument("--train_truths", action="store",
+                        default='data/train_Y.npy',
+                        help='training truth data filenames')
+
+    parser.add_argument("--valid_features", action="store",
+                        default='data/test_X.npy',
+                        help='validation feature data filenames')
+    parser.add_argument("--valid_truths", action="store",
+                        default='data/test_Y.npy',
+                        help='validation truth data filenames')
+
+    parser.add_argument("--output_files", action="store",
+                        default='result.csv',
+                        help="output filename")
+
+    # parser.add_argument("--shared_nnet_spec", action="store",
+                        # default=argparse.SUPPRESS, # DEF_SHARED_NNET_SPEC,
+                        # help='network structure of shared layer')
+    # parser.add_argument("--individual_nnet_spec", action="store",
+                        # default=argparse.SUPPRESS, # DEF_INDIV_NNET_SPEC,
+                        # help='network structore of task-specific layer')
+
+    return parser
+
+
+def read_config_file(File):
+    config=configparser.ConfigParser()
+    config.read(File)
+    section=config.sections()
+    Global_Params={}
+
+    Global_Params['learning_rate'] =eval(config.get(section[0],'learning_rate'))
+    Global_Params['batch_size']    =eval(config.get(section[0],'batch_size'))
+    Global_Params['epochs']        =eval(config.get(section[0],'epochs'))
+    Global_Params['dropout']       =eval(config.get(section[0],'dropout'))
+
+    Global_Params['optimizer']     =eval(config.get(section[0],'optimizer'))
+
+    Global_Params['wv_len']          =eval(config.get(section[0],'wv_len'))
+    Global_Params['filter_sizes']          =eval(config.get(section[0],'filter_sizes'))
+    Global_Params['filter_sets']          =eval(config.get(section[0],'filter_sets'))
+    Global_Params['num_filters']          =eval(config.get(section[0],'num_filters'))
+
+    Global_Params['emb_l2']          =eval(config.get(section[0],'emb_l2'))
+    Global_Params['w_l2']          =eval(config.get(section[0],'w_l2'))
+
+
+    # note 'cool' is a boolean
+    #Global_Params['cool']          =config.get(section[0],'cool')
+    return Global_Params

Pilot3/P3B3/p3b3_baseline_keras2.py

@@ -0,0 +1,186 @@
+from __future__ import print_function
+import numpy as np
+import os, sys, gzip
+import urllib, zipfile
+TIMEOUT=1800 # in sec; set this to -1 for no timeout
+
+import keras
+from keras import backend as K
+import math
+from keras.layers.core import Dense, Dropout
+from keras import optimizers
+
+from keras.layers import Input
+from keras.models import Model
+
+from sklearn.metrics import f1_score
+
+import argparse
+
+import p3b3
+import p3_common as p3c
+import p3_common_keras as p3ck
+from solr_keras import CandleRemoteMonitor, compute_trainable_params, TerminateOnTimeOut
+
+import keras_mt_shared_cnn
+
+
+
+def get_p3b3_parser():
+        parser = argparse.ArgumentParser(prog='p3b3_baseline',
+            formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+            description='Multi-task CNN for data extraction from clinical reports - Pilot 3 Benchmark 3')
+
+        return p3b3.common_parser(parser)
+
+def initialize_parameters():
+    parser = get_p3b3_parser()
+    args = parser.parse_args()
+    print('Args', args)
+
+    GP=p3b3.read_config_file(args.config_file)
+    print(GP)
+
+    GP = p3c.args_overwrite_config(args, GP)
+    return GP
+
+
+
+def run_cnn( GP, train_x, train_y, test_x, test_y,
+    learning_rate = 0.01,
+    batch_size = 10,
+    epochs = 10,
+    dropout = 0.5,
+    optimizer = 'adam',
+    wv_len = 300,
+    filter_sizes = [3,4,5],
+    num_filters = [300,300,300],
+    emb_l2 = 0.001,
+    w_l2 = 0.01
+    ):
+
+    max_vocab = np.max( train_x )
+    max_vocab2 = np.max( test_x )
+    if max_vocab2 > max_vocab:
+        max_vocab = max_vocab2
+
+    wv_mat = np.random.randn( max_vocab + 1, wv_len ).astype( 'float32' ) * 0.1
+
+    num_classes = []
+    num_classes.append( np.max( train_y[ :, 0 ] ) + 1 )
+    num_classes.append( np.max( train_y[ :, 1 ] ) + 1 )
+    num_classes.append( np.max( train_y[ :, 2 ] ) + 1 )
+    num_classes.append( np.max( train_y[ :, 3 ] ) + 1 )
+
+
+    kerasDefaults = p3c.keras_default_config()
+    optimizer = p3ck.build_optimizer( optimizer, learning_rate, kerasDefaults )
+
+
+    cnn = keras_mt_shared_cnn.init_export_network(
+        num_classes= num_classes,
+        in_seq_len= 1500,
+        vocab_size= len( wv_mat ),
+        wv_space= wv_len,
+        filter_sizes= filter_sizes,
+        num_filters= num_filters,
+        concat_dropout_prob = dropout,
+        emb_l2= emb_l2,
+        w_l2= w_l2,
+        optimizer= optimizer )
+
+    print( cnn.summary() )
+
+    validation_data = ( { 'Input': test_x },
+        { 'Dense0': test_y[ :, 0 ],
+          'Dense1': test_y[ :, 1 ],
+          'Dense2': test_y[ :, 2 ],
+          'Dense3': test_y[ :, 3 ] } )
+
+    candleRemoteMonitor = CandleRemoteMonitor(params= GP)
+    timeoutMonitor = TerminateOnTimeOut(TIMEOUT)
+
+    history = cnn.fit(
+        x= np.array( train_x ),
+        y= [ np.array( train_y[ :, 0 ] ),
+             np.array( train_y[ :, 1 ] ),
+             np.array( train_y[ :, 2 ] ),
+             np.array( train_y[ :, 3 ] ) ],
+        batch_size= batch_size,
+        epochs= epochs,
+        verbose= 2,
+        validation_data= validation_data,
+        callbacks = [candleRemoteMonitor, timeoutMonitor]
+     )
+
+    return history
+
+
+def run( GP ):
+    filter_sizes = []
+    num_filters = []
+
+    start = GP[ 'filter_sizes' ]
+    end = start + GP[ 'filter_sets' ] 
+    n_filters = GP[ 'num_filters' ]
+    for k in range( start, end ):
+        filter_sizes.append( k )
+        num_filters.append( n_filters )
+
+    learning_rate = GP[ 'learning_rate' ]
+    batch_size = GP[ 'batch_size' ]
+    epochs = GP[ 'epochs' ]
+    dropout = GP[ 'dropout' ]
+    optimizer = GP[ 'optimizer' ]
+
+    wv_len = GP[ 'wv_len' ]
+    emb_l2 = GP[ 'emb_l2' ]
+    w_l2 = GP[ 'w_l2' ]
+
+    
+    '''
+    ## Read files
+    file_path = os.path.dirname(os.path.realpath(__file__))
+    print(file_path)
+    lib_path = os.path.abspath(os.path.join(file_path, '..', '..', 'common'))
+    sys.path.append(lib_path)
+
+    from data_utils import get_file
+    origin = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P3B1/P3B1_data.tar.gz'
+    data_set = 'P3B1_data'
+    data_path = get_file(data_set, origin, untar=True, md5_hash=None, cache_subdir='P3B1')
+
+    print('Data downloaded and stored at: ' + os.path.dirname(data_path))
+    print('Data path:' + data_path)
+    '''
+    data_path = '/lustre/atlas/proj-shared/csc249/yoonh/Benchmarks/Data/Pilot3'
+
+    train_x = np.load( data_path + '/train_X.npy' )
+    train_y = np.load( data_path + '/train_Y.npy' )
+    test_x = np.load( data_path + '/test_X.npy' )
+    test_y = np.load( data_path + '/test_Y.npy' )
+
+
+    ret = run_cnn(
+        GP, 
+        train_x, train_y, test_x, test_y,
+        learning_rate = learning_rate,
+        batch_size = batch_size,
+        epochs = epochs,
+        dropout = dropout,
+        optimizer = optimizer,
+        wv_len = wv_len,
+        filter_sizes = filter_sizes,
+        num_filters = num_filters,
+        emb_l2 = emb_l2,
+        w_l2 = w_l2       
+    )
+
+    print( 'Average loss:', str( ret.history['val_loss'] ) )
+    return ret
+
+
+if __name__  == "__main__":
+    gParameters=initialize_parameters()
+    avg_loss = run(gParameters)
+

Pilot3/P3B3/p3b3_default_model.txt

@@ -0,0 +1,13 @@
+[Global_Params]
+learning_rate = 0.01
+batch_size = 10
+epochs = 10
+dropout = 0.5
+optimizer = 'adam'
+wv_len = 300
+filter_sizes = 3
+filter_sets = 3
+num_filters = 300
+emb_l2 = 0.001
+w_l2 = 0.01
+