Merge branch 'master' of github.com:ECP-CANDLE/Benchmarks

Author: bvanessen

P1B3/p1b3.py

@@ -2,6 +2,7 @@
 from data_utils import get_file
 # from six.moves import cPickle
 
+import collections
 import gzip
 import logging
 import os
@@ -20,7 +21,7 @@
 
 logger = logging.getLogger(__name__)
 
-SEED = 2016
+SEED = 2017
 
 np.set_printoptions(threshold=np.nan)
 np.random.seed(SEED)
@@ -238,8 +239,8 @@ def load_dose_response(path, min_logconc=-5., max_logconc=-5., subsample=None):
     return df
 
 
-class RegressionDataGenerator(object):
-    """Generate merged drug response, drug descriptors and cell line essay data
+class DataLoader(object):
+    """Load merged drug response, drug descriptors and cell line essay data
     """
 
     def __init__(self, val_split=0.2, shuffle=True, drug_features='descriptors',
@@ -273,7 +274,6 @@ def __init__(self, val_split=0.2, shuffle=True, drug_features='descriptors',
             growth thresholds seperating non-response and response categories
         """
 
-        self.lock = threading.Lock()
         self.drug_features = drug_features
 
         server = 'http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/P1B3/'
@@ -349,62 +349,110 @@ def __init__(self, val_split=0.2, shuffle=True, drug_features='descriptors',
                             format(i, count, count/len(growth), lower, upper))
             logger.info('  Total: {:9d}'.format(len(growth)))
 
-        nrows = df_train_val.shape[0]
+        self.total = df_train_val.shape[0]
         self.n_test = df_test.shape[0]
-        self.n_val = int(nrows * val_split)
-        self.n_train = nrows - self.n_val
-
-        self.cycle_train = cycle(range(nrows - self.n_val))
-        self.cycle_val = cycle(range(nrows)[-self.n_val:])
-        self.cycle_test = cycle(range(nrows, nrows + self.n_test))
+        self.n_val = int(self.total * val_split)
+        self.n_train = self.total - self.n_val
         logger.info('Rows in train: {}, val: {}, test: {}'.format(self.n_train, self.n_val, self.n_test))
 
-        self.input_dim = self.df_cellline.shape[1] - 1 + 1  # remove CELLNAME; add concentration
-        logger.info('Features:')
-        logger.info('  concentration: 1')
-        logger.info('  cell line expression: {}'.format(self.input_dim-1))
+        self.input_shapes = collections.OrderedDict()
+        self.input_shapes['drug_concentration'] = (1,)
+        self.input_shapes['cellline_expression'] = (self.df_cellline.shape[1] - 1,)
         if self.drug_features in ['descriptors', 'both']:
-            self.input_dim += self.df_drug_desc.shape[1] - 1  # remove NSC
-            logger.info('  drug descriptors: {}'.format(self.df_drug_desc.shape[1] - 1))
+            self.input_shapes['drug_descriptors'] =  (self.df_drug_desc.shape[1] - 1,)  # remove NSC
         if self.drug_features in ['latent', 'both']:
-            self.input_dim += self.df_drug_auen.shape[1] - 1  # remove NSC
-            logger.info('  drug latent representations: {}'.format(self.df_drug_auen.shape[1] - 1))
+            self.input_shapes['drug_SMILES_latent'] = (self.df_drug_auen.shape[1] - 1,)  # remove NSC
         if self.drug_features == 'noise':
-            self.input_dim += self.df_drug_rand.shape[1] - 1  # remove NSC
-            logger.info('  drug random vectors: {}'.format(self.df_drug_rand.shape[1] - 1))
+            self.input_shapes['drug_random_vector'] = (self.df_drug_rand.shape[1] - 1,)  # remove NSC
+
+        logger.info('Input features shapes:')
+        for k, v in self.input_shapes.items():
+            logger.info('  {}: {}'.format(k, v))
+
+        self.input_dim = sum([np.prod(x) for x in self.input_shapes.values()])
         logger.info('Total input dimensions: {}'.format(self.input_dim))
 
-    def flow(self, batch_size=32, data='train', topology=None):
-        if data == 'val':
-            cyc = self.cycle_val
-        elif data == 'test':
-            cyc = self.cycle_test
+
+class DataGenerator(object):
+    """Generate training, validation or testing batches from loaded data
+    """
+
+    def __init__(self, data, partition='train', batch_size=32, shape=None, concat=True):
+        """Initialize data
+
+        Parameters
+        ----------
+        data: DataLoader object
+            loaded data object containing original data frames for molecular, drug and response data
+        partition: 'train', 'val', or 'test'
+            partition of data to generate for
+        batch_size: integer (default 32)
+            batch size of generated data
+        shape: None, '1d' or 'add_1d' (default None)
+            keep original feature shapes, make them flat or add one extra dimension (for convolution or locally connected layers in some frameworks)
+        concat: True or False (default True)
+            concatenate all features if set to True
+        """
+        self.lock = threading.Lock()
+        self.data = data
+        self.partition = partition
+        self.batch_size = batch_size
+        self.shape = shape
+        self.concat = concat
+
+        if partition == 'train':
+            self.cycle = cycle(range(data.n_train))
+            self.num_data = data.n_train
+        elif partition == 'val':
+            self.cycle = cycle(range(data.total)[-data.n_val:])
+            self.num_data = data.n_val
+        elif partition == 'test':
+            self.cycle = cycle(range(data.total, data.total + data.n_test))
+            self.num_data = data.n_test
         else:
-            cyc = self.cycle_train
+            raise Exception('Data partition "{}" not recognized.'.format(partition))
 
+    def flow(self):
+        """Keep generating data batches
+        """
         while 1:
             self.lock.acquire()
-            indices = list(islice(cyc, batch_size))
+            indices = list(islice(self.cycle, self.batch_size))
             # print("\nProcess: {}, Batch indices start: {}".format(multiprocessing.current_process().name, indices[0]))
             self.lock.release()
 
-            df = self.df_response.iloc[indices, :]
-            df = pd.merge(df, self.df_cellline, on='CELLNAME')
+            df = self.data.df_response.iloc[indices, :]
+            df = pd.merge(df, self.data.df_cellline, on='CELLNAME')
 
-            if self.drug_features in ['descriptors', 'both']:
-                df = df.merge(self.df_drug_desc, on='NSC')
-            if self.drug_features in ['latent', 'both']:
-                df = df.merge(self.df_drug_auen, on='NSC')
-            if self.drug_features == 'noise':
-                df = df.merge(self.df_drug_rand, on='NSC')
+            if self.data.drug_features in ['descriptors', 'both']:
+                df = df.merge(self.data.df_drug_desc, on='NSC')
+            if self.data.drug_features in ['latent', 'both']:
+                df = df.merge(self.data.df_drug_auen, on='NSC')
+            if self.data.drug_features == 'noise':
+                df = df.merge(self.data.df_drug_rand, on='NSC')
 
             df = df.drop(['CELLNAME', 'NSC'], 1)
             x = np.array(df.iloc[:, 1:])
             y = np.array(df.iloc[:, 0])
             y = y / 100.
 
-            if topology == 'simple_local':
-                yield x.reshape(x.shape + (1,)), y
-                # yield x.reshape(x.shape[0], 1, x.shape[1]), y
+            if self.concat:
+                if self.shape == 'add_1d':
+                    yield x.reshape(x.shape + (1,)), y
+                else:
+                    yield x, y
             else:
-                yield x, y
+                x_list = []
+                index = 0
+                for v in self.data.input_shapes.values():
+                    length = np.prod(v)
+                    subset = x[:, index:index+length]
+                    if self.shape == '1d':
+                        reshape = (x.shape[0], length)
+                    elif self.shape == 'add_1d':
+                        reshape = (x.shape[0],) + v + (1,)
+                    else:
+                        reshape = (x.shape[0],) + v
+                    x_list.append(subset.reshape(reshape))
+                    index += length
+                yield x_list, y

P1B3/p1b3_baseline.py

@@ -47,7 +47,8 @@
 # Activation function (options: 'relu', 'tanh', 'sigmoid', 'hard_sigmoid', 'linear')
 ACTIVATION = 'relu'
 LOSS = 'mse'
-OPTIMIZER = 'adam'
+OPTIMIZER = 'sgd'
+# OPTIMIZER = 'adam'
 
 # Type of feature scaling (options: 'maxabs': to [-1,1]
 #                                   'minmax': to [0,1]
@@ -65,11 +66,11 @@
 DENSE_LAYERS = [D1, D2, D3, D4]
 
 # Number of units per locally connected layer
-LC1 = 10, 10        # nb_filter, filter_length
-LC2 = 0, 0         # disabled layer
-# LOCALLY_CONNECTED_LAYERS = list(LC1 + LC2)
-LOCALLY_CONNECTED_LAYERS = [0, 0]
-POOL = 100
+C1 = 10, 10, 5       # nb_filter, filter_length, stride
+C2 = 0, 0, 0         # disabled layer
+# CONVOLUTION_LAYERS = list(C1 + C2)
+CONVOLUTION_LAYERS = [0, 0, 0]
+POOL = 10
 
 MIN_LOGCONC = -5.
 MAX_LOGCONC = -4.
@@ -91,18 +92,18 @@ def get_parser():
     parser.add_argument("-b", "--batch_size", action="store",
                         default=BATCH_SIZE, type=int,
                         help="batch size")
-    parser.add_argument("-c", "--convolution", action="store_true",
-                        default=False,
-                        help="use convolution layers instead of locally connected layers")
+    parser.add_argument("-c", "--convolution", action="store", nargs='+', type=int,
+                        default=CONVOLUTION_LAYERS,
+                        help="integer array describing convolution layers: conv1_nb_filter, conv1_filter_len, conv1_stride, conv2_nb_filter, conv2_filter_len, conv2_stride ...")
     parser.add_argument("-d", "--dense", action="store", nargs='+', type=int,
                         default=DENSE_LAYERS,
                         help="number of units in fully connected layers in an integer array")
     parser.add_argument("-e", "--epochs", action="store",
                         default=NB_EPOCH, type=int,
                         help="number of training epochs")
-    parser.add_argument("-l", "--locally_connected", action="store", nargs='+', type=int,
-                        default=LOCALLY_CONNECTED_LAYERS,
-                        help="integer array describing locally connected layers: layer1_nb_filter, layer1_filter_len, layer2_nb_filter, layer2_filter_len, ...")
+    parser.add_argument("-l", "--locally_connected", action="store_true",
+                        default=False,
+                        help="use locally connected layers instead of convolution layers")
     parser.add_argument("-o", "--optimizer", action="store",
                         default=OPTIMIZER,
                         help="keras optimizer to use: sgd, rmsprop, ...")
@@ -163,15 +164,16 @@ def extension_from_parameters(args):
     ext += '.E={}'.format(args.epochs)
     if args.feature_subsample:
         ext += '.F={}'.format(args.feature_subsample)
-    if args.locally_connected:
-        name = 'C' if args.convolution else 'LC'
-        layer_list = list(range(0, len(args.locally_connected), 2))
+    if args.convolution:
+        name = 'LC' if args.locally_connected else 'C'
+        layer_list = list(range(0, len(args.convolution), 3))
         for l, i in enumerate(layer_list):
-            nb_filter = args.locally_connected[i]
-            filter_len = args.locally_connected[i+1]
-            if nb_filter <= 0 or filter_len <= 0:
+            nb_filter = args.convolution[i]
+            filter_len = args.convolution[i+1]
+            stride = args.convolution[i+2]
+            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                 break
-            ext += '.{}{}={},{}'.format(name, l+1, nb_filter, filter_len)
+            ext += '.{}{}={},{},{}'.format(name, l+1, nb_filter, filter_len, stride)
         if args.pool and layer_list[0] and layer_list[1]:
             ext += '.P={}'.format(args.pool)
     for i, n in enumerate(args.dense):
@@ -308,53 +310,54 @@ def main():
 
     ext = extension_from_parameters(args)
 
-    datagen = p1b3.RegressionDataGenerator(feature_subsample=args.feature_subsample,
-                                           scaling=args.scaling,
-                                           drug_features=args.drug_features,
-                                           scramble=args.scramble,
-                                           min_logconc=args.min_logconc,
-                                           max_logconc=args.max_logconc,
-                                           subsample=args.subsample,
-                                           category_cutoffs=args.category_cutoffs)
+    loader = p1b3.DataLoader(feature_subsample=args.feature_subsample,
+                             scaling=args.scaling,
+                             drug_features=args.drug_features,
+                             scramble=args.scramble,
+                             min_logconc=args.min_logconc,
+                             max_logconc=args.max_logconc,
+                             subsample=args.subsample,
+                             category_cutoffs=args.category_cutoffs)
 
-    topology = 'dense'
+    gen_shape = None
     out_dim = 1
 
     model = Sequential()
-    if args.locally_connected and args.locally_connected[0]:
-        topology = 'simple_local'
-        layer_list = list(range(0, len(args.locally_connected), 2))
+    if args.convolution and args.convolution[0]:
+        gen_shape = 'add_1d'
+        layer_list = list(range(0, len(args.convolution), 3))
         for l, i in enumerate(layer_list):
-            nb_filter = args.locally_connected[i]
-            filter_len = args.locally_connected[i+1]
-            if nb_filter <= 0 or filter_len <= 0:
+            nb_filter = args.convolution[i]
+            filter_len = args.convolution[i+1]
+            stride = args.convolution[i+2]
+            if nb_filter <= 0 or filter_len <= 0 or stride <= 0:
                 break
-            if args.convolution:
-                model.add(Convolution1D(nb_filter, filter_len, input_shape=(datagen.input_dim, 1), activation=args.activation))
+            if args.locally_connected:
+                model.add(LocallyConnected1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
             else:
-                model.add(LocallyConnected1D(nb_filter, filter_len, input_shape=(datagen.input_dim, 1), activation=args.activation))
+                model.add(Convolution1D(nb_filter, filter_len, subsample_length=stride, input_shape=(loader.input_dim, 1), activation=args.activation))
             if args.pool:
                 model.add(MaxPooling1D(pool_length=args.pool))
         model.add(Flatten())
 
     for layer in args.dense:
         if layer:
-            model.add(Dense(layer, input_dim=datagen.input_dim, activation=args.activation))
+            model.add(Dense(layer, input_dim=loader.input_dim, activation=args.activation))
             if args.drop:
                 model.add(Dropout(args.drop))
     model.add(Dense(out_dim))
 
     model.summary()
     model.compile(loss=args.loss, optimizer=args.optimizer)
 
-    train_gen = datagen.flow(batch_size=args.batch_size, topology=topology)
-    val_gen = datagen.flow(data='val', batch_size=args.batch_size, topology=topology)
-    val_gen2 = datagen.flow(data='val', batch_size=args.batch_size, topology=topology)
-    test_gen = datagen.flow(data='test', batch_size=args.batch_size, topology=topology)
+    train_gen = p1b3.DataGenerator(loader, batch_size=args.batch_size, shape=gen_shape).flow()
+    val_gen = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
+    val_gen2 = p1b3.DataGenerator(loader, partition='val', batch_size=args.batch_size, shape=gen_shape).flow()
+    test_gen = p1b3.DataGenerator(loader, partition='test', batch_size=args.batch_size, shape=gen_shape).flow()
 
-    train_samples = int(datagen.n_train/args.batch_size) * args.batch_size
-    val_samples = int(datagen.n_val/args.batch_size) * args.batch_size
-    test_samples = int(datagen.n_test/args.batch_size) * args.batch_size
+    train_samples = int(loader.n_train/args.batch_size) * args.batch_size
+    val_samples = int(loader.n_val/args.batch_size) * args.batch_size
+    test_samples = int(loader.n_test/args.batch_size) * args.batch_size
 
     train_samples = args.train_samples if args.train_samples else train_samples
     val_samples = args.val_samples if args.val_samples else val_samples