Added the P1B2 benchmark.

Author: bvanessen

P2B1/__init__.py

@@ -0,0 +1,4 @@
+print 'import candle'
+import os,sys
+HOME=os.environ['HOME']
+sys.path.append('%s/Work/Python/Git_Folder/caffe-tools/Newkeras/keras'%HOME)

P2B1/__main__.py

@@ -0,0 +1,124 @@
+import theano
+import numpy as np
+import scipy as sp
+import pickle
+import sys,os
+import glob
+import optparse
+HOME=os.environ['HOME']
+def parse_list(option, opt, value, parser):
+  setattr(parser.values, option.dest, value.split(','))
+
+if __name__=="__main__":
+### Hyperparameters and model save path
+	parser=optparse.OptionParser()
+	parser.add_option("--train", action="store_true",dest="train_bool",default=False,help="Invoke training")
+	parser.add_option("--learning-rate",help="learning rate",dest="learning_rate",type=float,default=0.1)
+	parser.add_option("--noise-factor",help="noise",dest="noise_factor",type=float,default=0.0)
+	parser.add_option("--cool",action="store_true",dest="cool",default=False,help="Cool Learning Rate")
+	parser.add_option("--epochs",help="epochs",dest="epochs",type=int,default=1)
+	parser.add_option("--home-dir",help="Home Directory",dest="home_dir",type=str,default='/Users/talathi1/Work/Python/Git_Folder/caffe-tools/keras')
+	parser.add_option("--save-dir",help="Save Directory",dest="save_path",type=str,default=None)
+	parser.add_option("--model-file",help="Trained Model Pickle File",dest="weight_path",type=str,default=None)
+	parser.add_option("--memo",help="Memo",dest="base_memo",type=str,default=None)
+	(opts,args)=parser.parse_args()
+
+## Example of training command:
+#python mnist_conv_autoencoder.py -C --save-dir /Users/talathi1/Work/Python/Models/Test_Models --memo test --epochs 1 --data-path /Users/talathi1/Work/DataSets/mnist.pkl.gz --learning-rate 0.01 --train --classify --unpool_type 3
+
+	if not os.path.isdir(opts.home_dir):
+		print ('Keras home directory not set')
+		sys.exit(0)
+	sys.path.append('home_dir')
+	
+	import candle.candle_helper_functions as hf
+	reload(hf)
+	maps=hf.autoencoder_preprocess()
+	#from keras.Helpermodules.mnist_autoencoder_helper_functions import mnist_conv_deconv_simple, mnist_conv_deconv_complex,mnist_autoencoder_preprocess,generate_figure
+	
+	from keras.optimizers import SGD,RMSprop
+	from keras.datasets import mnist
+	from keras.callbacks import LearningRateScheduler,ModelCheckpoint
+	from keras.regularizers import l2,WeightRegularizer
+	from keras import callbacks
+	from keras.layers.advanced_activations import ELU
+	from keras.preprocessing.image import ImageDataGenerator
+
+	batch_size = 16
+##### Read Data ########
+	print ('Reading Data...')
+	data_file='%s/Work/DataSets/CANDLE/sim-numpy.npy'%HOME ### can code to read at the terminal
+	print 'Data File: %s' %data_file
+	print 'Data Format: [Num Samples, Num Molecules, Num Atoms, Position]'
+	
+	X=np.load(data_file) ### Data is: Samples, Molecules, Atoms, x-pos,y-pos,z-pos
+	## Take center of mass for atoms:
+	X_A=X.mean(axis=2) ## Data is: Samples, Molecules, x-pos,y-pos,z-pos
+	#X_train=X_A.reshape(X_A.shape[0],X_A.shape[1]*X_A.shape[2])
+	X_train=X_A[:,:,2] ## only consider z-dimension
+	y_train=X_train.copy()
+	input_dim=X_train.shape[1]
+	mu, sigma = np.mean(X_train), np.std(X_train)
+	mu=0.0;sigma=1.0
+	X_train=maps.renormalize(X_train,mu,sigma)
+	datagen=hf.ImageNoiseDataGenerator(corruption_level=opts.noise_factor)  ## Add some corruption to input data ## idead for denoising auto encoder 
+		
+	print('X_train type and shape:', X_train.dtype, X_train.shape)
+	print('X_train.min():', X_train.min())
+	print('X_train.max():', X_train.max())
+
+### Define Model, Solver and Compile ##########
+	print ('Define the model and compile')
+	opt = SGD(lr=opts.learning_rate, decay=0.0, momentum=0.975, nesterov=True)
+	
+	print ('using mlp network')
+	model_type='mlp'
+	hidden_layers=[512,256,128,64,32,16]
+	model=hf.dense_auto(weights_path=opts.weight_path,input_shape=(input_dim,),nonlinearity='elu',hidden_layers=hidden_layers)
+		
+	memo='%s_%s_%0.5f'%(opts.base_memo,model_type,opts.learning_rate)
+
+	print 'Autoencoder Regression problem'
+	model.compile(optimizer='adadelta', loss='mean_squared_error')
+
+#### Print Model Configuration ###########
+	num_layers=len(model.layers)
+	print '*'*10,'Model Configuration','*'*10
+	for i  in range(len(model.layers)):	
+		print i,': ',model.layers[i].name, ':', model.layers[i].output_shape[:]
+
+### Set up for Training and Validation
+	total_epochs = opts.epochs
+	initial_lrate=opts.learning_rate
+	if opts.cool:
+		drop=0.5
+	else:
+		drop=1.0
+	
+	epochs_drop=1+int(np.floor(total_epochs/3))
+		
+	def step_decay(epoch):
+		global initial_lrate,epochs_drop,drop
+		lrate = initial_lrate * np.power(drop, np.floor((1+epoch)/epochs_drop))
+		return lrate
+	lr_scheduler = LearningRateScheduler(step_decay)
+
+#### Train the Model
+	if opts.train_bool:
+		history = callbacks.History()
+		if opts.save_path !=None:
+			model_file='%s/%s.hdf5'%(opts.save_path,memo)
+			checkpointer=ModelCheckpoint(filepath=model_file, verbose=1)
+			callbacks=[history,lr_scheduler,checkpointer]
+		else:
+			callbacks=[history,lr_scheduler]
+		model.fit_generator(datagen.flow(X_train, y_train, batch_size=batch_size),\
+			samples_per_epoch=X_train.shape[0],nb_epoch=total_epochs,callbacks=callbacks,verbose=1)
+		loss_data={'train': history.history['loss']}
+		if opts.save_path!=None:
+			loss_file='%s/%s.pkl'%(opts.save_path,memo)
+			o=open(loss_file,'wb')
+			pickle.dump(loss_data,o)
+			o.close()
+	
+	

P2B1/candle_helper_functions.py

@@ -0,0 +1,181 @@
+from __future__ import absolute_import
+import theano
+import matplotlib
+if 'MACOSX' in matplotlib.get_backend().upper():
+  matplotlib.use('TKAgg')
+import pylab as py
+py.ion() ## Turn on plot visualization
+
+import gzip,pickle
+import numpy as np
+from PIL import Image
+import cv2
+import keras.backend as K
+K.set_image_dim_ordering('th')
+from keras.layers import Input
+from keras.models import Sequential,Model
+from keras.layers.core import Flatten, Dense, Dropout, Activation, Reshape
+from keras.layers.convolutional import Convolution2D, MaxPooling2D,Convolution1D
+from keras.layers.convolutional import ZeroPadding2D,UpSampling2D,Unpooling2D,perforated_Unpooling2D,DePool2D
+from keras.initializations import normal, identity, he_normal,glorot_normal,glorot_uniform,he_uniform
+from keras.layers.normalization import BatchNormalization
+import threading
+
+############# Define Data Generators ################
+class ImageNoiseDataGenerator(object):
+    '''Generate minibatches with
+    realtime data augmentation.
+    '''
+    def __init__(self,corruption_level=0.5):
+
+        self.__dict__.update(locals())
+        self.p=corruption_level
+        self.lock = threading.Lock()
+
+    def _flow_index(self, N, batch_size=32, shuffle=False, seed=None):
+        b = 0
+        total_b = 0
+        while 1:
+            if b == 0:
+                if seed is not None:
+                    np.random.seed(seed + total_b)
+
+                if shuffle:
+                    index_array = np.random.permutation(N)
+                else:
+                    index_array = np.arange(N)
+
+            current_index = (b * batch_size) % N
+            if N >= current_index + batch_size:
+                current_batch_size = batch_size
+            else:
+                current_batch_size = N - current_index
+
+            if current_batch_size == batch_size:
+                b += 1
+            else:
+                b = 0
+            total_b += 1
+            yield index_array[current_index: current_index + current_batch_size], current_index, current_batch_size
+
+    def flow(self, X, y, batch_size=32, shuffle=False, seed=None):
+        assert len(X) == len(y)
+        self.X = X
+        self.y = y
+        self.flow_generator = self._flow_index(X.shape[0], batch_size, shuffle, seed)
+        return self
+
+    def __iter__(self):
+        # needed if we want to do something like for x,y in data_gen.flow(...):
+        return self
+
+    def next(self):
+        # for python 2.x
+        # Keep under lock only the mechainsem which advance the indexing of each batch
+        # see # http://anandology.com/blog/using-iterators-and-generators/
+        with self.lock:
+            index_array, current_index, current_batch_size = next(self.flow_generator)
+        # The transformation of images is not under thread lock so it can be done in parallel
+        bX = np.zeros(tuple([current_batch_size] + list(self.X.shape)[1:]))
+        for i, j in enumerate(index_array):
+            x = self.X[j]
+            x = self.insertnoise(x,corruption_level=self.p)
+            bX[i] = x
+        bY = self.y[index_array]
+        return bX, bY
+
+    def __next__(self):
+        # for python 3.x
+        return self.next()
+
+    def insertnoise(self,x,corruption_level=0.5):
+        return np.random.binomial(1,1-corruption_level,x.shape)*x
+
+##### Define Neural Network Models ###################
+def dense_auto(weights_path=None,input_shape=(784,),hidden_layers=None,nonlinearity='relu'):
+    input_img = Input(shape=input_shape)
+    
+    if hidden_layers!=None:
+        if type(hidden_layers)!=list:
+            hidden_layers=list(hidden_layers)
+        for i,l in enumerate(hidden_layers):
+            if i==0: 
+                encoded=Dense(l,activation=nonlinearity)(input_img)
+            else:
+                encoded=Dense(l,activation=nonlinearity)(encoded)
+
+        for i,l in reversed(list(enumerate(hidden_layers))):
+            if i <len(hidden_layers)-1:
+                if i==len(hidden_layers)-2:
+                    decoded=Dense(l,activation=nonlinearity)(encoded)
+                else:
+                    decoded=Dense(l,activation=nonlinearity)(decoded)
+        decoded=Dense(input_shape[0])(decoded)
+    else:
+        decoded=Dense(input_shape[0])(input_img)
+
+    model=Model(input=input_img,output=decoded)
+    
+    if weights_path:
+        print('Loading Model')
+        model.load_weights(weights_path)
+    return model
+
+def dense_simple(weights_path=None,input_shape=(784,),nonlinearity='relu'):
+    model=Sequential()
+    ## encoder
+    model.add(Dense(512,input_shape=input_shape,activation=nonlinearity))
+    BatchNormalization()
+    model.add(Dense(256,activation=nonlinearity))
+    BatchNormalization()
+    model.add(Dense(128,activation=nonlinearity))
+    BatchNormalization()
+    model.add(Dense(64,activation=nonlinearity))
+    BatchNormalization()
+    model.add(Dense(32,activation=nonlinearity))
+    BatchNormalization()
+    model.add(Dense(16,activation=nonlinearity))
+    BatchNormalization()
+    ## decoder
+    model.add(Dense(32))
+    BatchNormalization()
+    model.add(Dense(64))
+    BatchNormalization()
+    model.add(Dense(128))
+    BatchNormalization()
+    model.add(Dense(256))
+    BatchNormalization()
+    model.add(Dense(512))
+    BatchNormalization()
+    model.add(Dense(input_shape[0],activation='linear'))    
+    if weights_path:
+        print('Loading Model')
+        model.load_weights(weights_path)
+    return model
+
+
+class autoencoder_preprocess():
+    def __init__(self,img_size=(784,),noise_factor=0.):
+        self.noise=noise_factor
+        self.img_size=img_size
+        self.lock = threading.Lock()
+
+    def add_noise(self,X_train):
+        ## Add noise to input data
+        np.random.seed(100)
+        ind=np.where(X_train==0)
+        rn=self.noise*np.random.rand(np.shape(ind)[1])
+        X_train[ind]=rn
+        return X_train
+    
+    def renormalize(self,X_train,mu,sigma):
+        X_train=(X_train-mu)/sigma
+        X_train = X_train.astype("float32")
+        return X_train
+
+def get_activations(model, layer, X_batch):
+    get_activations = K.function([model.layers[0].input, K.learning_phase()], model.layers[layer].output)
+    activations = get_activations([X_batch,0])
+    return activations
+
+