Add simple Optimizer example based on MNIST.

Author: lucasb-eyer

examples/Optimizers/mnist.py

@@ -0,0 +1,29 @@
+import os
+import gzip
+import pickle
+import sys
+
+# Python 2/3 compatibility.
+try:
+    from urllib.request import urlretrieve
+except ImportError:
+    from urllib import urlretrieve
+
+
+'''Adapted from theano tutorial'''
+
+
+def load_mnist(data_file = os.path.join(os.path.dirname(__file__), 'mnist.pkl.gz')):
+
+    if not os.path.exists(data_file):
+        origin = 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
+        print('Downloading data from {}'.format(origin))
+        urlretrieve(origin, data_file)
+
+    print('... loading data')
+
+    with gzip.open(data_file, 'rb') as f:
+        if sys.version_info[0] == 3:
+            return pickle.load(f, encoding='latin1')
+        else:
+            return pickle.load(f)

examples/Optimizers/model.py

@@ -0,0 +1,65 @@
+import DeepFried2 as df
+
+
+def net():
+    model = df.Sequential()
+    model.add(df.Linear(28*28, 100))
+    model.add(df.ReLU())
+
+    model.add(df.Linear(100, 100))
+    model.add(df.ReLU())
+
+    model.add(df.Linear(100, 100))
+    model.add(df.ReLU())
+
+    model.add(df.Linear(100, 10))
+    model.add(df.SoftMax())
+    return model
+
+
+def lenet():
+    model = df.Sequential()
+    model.add(df.Reshape(-1, 1, 28, 28))
+    model.add(df.SpatialConvolutionCUDNN(1, 32, 5, 5, 1, 1, 2, 2, with_bias=False))
+    model.add(df.BatchNormalization(32))
+    model.add(df.ReLU())
+    model.add(df.SpatialMaxPoolingCUDNN(2, 2))
+
+    model.add(df.SpatialConvolutionCUDNN(32, 64, 5, 5, 1, 1, 2, 2, with_bias=False))
+    model.add(df.BatchNormalization(64))
+    model.add(df.ReLU())
+    model.add(df.SpatialMaxPoolingCUDNN(2, 2))
+    model.add(df.Reshape(-1, 7*7*64))
+
+    model.add(df.Linear(7*7*64, 100, with_bias=False))
+    model.add(df.BatchNormalization(100))
+    model.add(df.ReLU())
+    model.add(df.Dropout(0.5))
+
+    model.add(df.Linear(100, 10))
+    model.add(df.SoftMax())
+    return model
+
+
+def lenet2():
+    model = df.Sequential()
+    model.add(df.Reshape(-1, 1, 28, 28))
+    model.add(df.SpatialConvolution(1, 32, 5, 5, 1, 1, with_bias=False))
+    model.add(df.BatchNormalization(32))
+    model.add(df.ReLU())
+    model.add(df.SpatialMaxPooling(2, 2))
+
+    model.add(df.SpatialConvolution(32, 64, 5, 5, 1, 1, with_bias=False))
+    model.add(df.BatchNormalization(64))
+    model.add(df.ReLU())
+    model.add(df.SpatialMaxPooling(2, 2))
+    model.add(df.Reshape(-1, 4*4*64))
+
+    model.add(df.Linear(4*4*64, 100, with_bias=False))
+    model.add(df.BatchNormalization(100))
+    model.add(df.ReLU())
+    model.add(df.Dropout(0.5))
+
+    model.add(df.Linear(100, 10))
+    model.add(df.SoftMax())
+    return model

examples/Optimizers/run.py

@@ -0,0 +1,48 @@
+import DeepFried2 as df
+
+from examples.utils import make_progressbar
+
+from mnist import load_mnist
+from train import train
+from test import test
+from model import net, lenet2
+
+
+if __name__ == "__main__":
+    print("THIS IS JUST AN EXAMPLE.")
+    print("Please don't take these numbers as a benchmark.")
+    print("While the optimizer's parameters have been grid-searched,")
+    print("a fair comparison would run all experiments multiple times AND RUN MORE THAN FIVE EPOCHS.")
+
+    batch_size = 64
+
+    (Xtrain, ytrain), (Xval, yval), (Xtest, ytest) = load_mnist()
+
+    criterion = df.ClassNLLCriterion()
+
+    def run(optim):
+        progress = make_progressbar('Training with ' + str(optim), 5)
+        progress.start()
+
+        model = net()
+        model.training()
+        for epoch in range(5):
+            train(Xtrain, ytrain, model, optim, criterion, batch_size, 'train')
+            train(Xtrain, ytrain, model, optim, criterion, batch_size, 'stats')
+            progress.update(epoch+1)
+
+        progress.finish()
+
+        model.evaluate()
+        nll, _ = test(Xtrain, ytrain, model, batch_size)
+        _, nerr = test(Xval, yval, model, batch_size)
+
+        print("Trainset NLL: {:.2f}".format(nll))
+        print("Testset errors: {}".format(nerr))
+
+    run(df.SGD(lr=1e-1))
+    run(df.Momentum(lr=1e-2, momentum=0.95))
+    run(df.Nesterov(lr=1e-2, momentum=0.90))
+    run(df.AdaGrad(lr=1e-2, eps=1e-4))
+    run(df.RMSProp(lr=1e-3, rho=0.90, eps=1e-5))
+    run(df.AdaDelta(rho=0.99, lr=5e-1, eps=1e-4))

examples/Optimizers/test.py

@@ -0,0 +1,21 @@
+import numpy as np
+import theano as th
+
+
+def test(X, y, model, batch_size):
+
+    nll = 0
+    nerrors = 0
+    for j in range((len(X) + batch_size - 1) // batch_size):
+        # Note: numpy correctly handles the size of the last minibatch.
+        miniX = X[j*batch_size : (j+1)*batch_size].astype(th.config.floatX)
+        miniy = y[j*batch_size : (j+1)*batch_size]
+
+        pred_probas = model.forward(miniX)
+        preds = np.argmax(pred_probas, axis=1)
+
+        nll -= sum(np.log(np.clip(pred_probas[np.arange(len(miniX)), miniy], 1e-15, 1-1e-15)))
+        nerrors += sum(preds != miniy)
+
+    #accuracy = 1 - float(nerrors)/len(X)
+    return nll, nerrors

examples/Optimizers/train.py

@@ -0,0 +1,21 @@
+import numpy as np
+import theano as th
+
+
+def train(X, y, model, optimiser, criterion, batch_size, mode='train'):
+
+    shuffle = np.random.permutation(len(X))
+
+    for j in range(len(X) // batch_size):
+        indices = shuffle[j*batch_size : (j+1)*batch_size]
+        mini_batch_input = X[indices].astype(th.config.floatX)
+        mini_batch_targets = y[indices].astype(th.config.floatX)
+
+        if mode == 'train':
+            model.zero_grad_parameters()
+            model.accumulate_gradients(mini_batch_input, mini_batch_targets, criterion)
+            optimiser.update_parameters(model)
+        elif mode == 'stats':
+            model.accumulate_statistics(mini_batch_input)
+        else:
+            assert False, "Mode should be either 'train' or 'stats'"