Merge pull request #5 from ikostrikov/master

Added convolutional layers and a lenet example for MNIST.

Author: lucasb-eyer

beacon8/layers/SpatialConvolutionCUDNN.py

@@ -0,0 +1,39 @@
+import theano as _th
+import numpy as _np
+import theano.sandbox.cuda.dnn as _dnn
+
+from .Module import Module
+
+
+class SpatialConvolutionCUDNN(Module):
+    def __init__(self, n_input_plane, n_output_plane, k_w, k_h, d_w=1, d_h=1, pad_w=0, pad_h=0, with_bias=True):
+        Module.__init__(self)
+        self.n_input_plane = n_input_plane
+        self.n_output_plane = n_output_plane
+        self.k_w = k_w
+        self.k_h = k_h
+        self.d_w = d_w
+        self.d_h = d_h
+        self.pad_w = pad_w
+        self.pad_h = pad_h
+        self.with_bias = with_bias
+        w_bound = _np.sqrt(4. / ((self.n_input_plane + self.n_output_plane) * self.k_w * self.k_h))
+
+        W = _np.random.uniform(low=-w_bound, high=w_bound, size=(n_output_plane, n_input_plane, k_h, k_w))
+        self.weight = _th.shared(W.astype(dtype=_th.config.floatX))
+        self.grad_weight = _th.shared((W*0).astype(dtype=_th.config.floatX))
+
+        if self.with_bias:
+            self.bias = _th.shared(_np.zeros(shape=(n_output_plane, ), dtype=_th.config.floatX))
+            self.grad_bias = _th.shared(_np.zeros(shape=(n_output_plane, ), dtype=_th.config.floatX))
+
+    def symb_forward(self, symb_input):
+        conv_output = _dnn.dnn_conv(img=symb_input,
+                                    kerns=self.weight,
+                                    border_mode=(self.pad_h, self.pad_w),
+                                    subsample=(self.d_h, self.d_w))
+
+        if self.with_bias:
+            return conv_output + self.bias.dimshuffle('x', 0, 'x', 'x')
+        else:
+            return conv_output

beacon8/layers/SpatialMaxPoolingCUDNN.py

@@ -0,0 +1,32 @@
+import theano.sandbox.cuda.dnn as _dnn
+
+from .Module import Module
+
+
+class SpatialMaxPoolingCUDNN(Module):
+    def __init__(self, k_w, k_h, d_w=None, d_h=None, pad_w=0, pad_h=0):
+        Module.__init__(self)
+        self.k_w = k_w
+        self.k_h = k_h
+
+        if d_w is None:
+            self.d_w = self.k_w
+        else:
+            self.d_w = d_w
+
+        if d_h is None:
+            self.d_h = self.k_h
+        else:
+            self.d_h = d_h
+
+        self.pad_w = pad_w
+        self.pad_h = pad_h
+
+    def symb_forward(self, symb_input):
+        return _dnn.dnn_pool(
+            img=symb_input,
+            ws=(self.k_w, self.k_h),
+            stride=(self.d_w, self.d_h),
+            mode='max',
+            pad=(self.pad_w, self.pad_h)
+        )

beacon8/layers/__init__.py

@@ -8,3 +8,5 @@
 from .AddConstant import *
 from .Log import *
 from .Reshape import *
+from .SpatialConvolutionCUDNN import *
+from .SpatialMaxPoolingCUDNN import *

examples/MNIST/model.py

@@ -16,3 +16,27 @@ def net():
     model.add(bb8.SoftMax())
     return model
 
+
+def lenet():
+    model = bb8.Sequential()
+    model.add(bb8.Reshape(-1, 1, 28, 28))
+    model.add(bb8.SpatialConvolutionCUDNN(1, 32, 5, 5, 1, 1, 2, 2, with_bias=False))
+    model.add(bb8.BatchNormalization(32))
+    model.add(bb8.ReLU())
+    model.add(bb8.SpatialMaxPoolingCUDNN(2, 2))
+
+    model.add(bb8.SpatialConvolutionCUDNN(32, 64, 5, 5, 1, 1, 2, 2, with_bias=False))
+    model.add(bb8.BatchNormalization(64))
+    model.add(bb8.ReLU())
+    model.add(bb8.SpatialMaxPoolingCUDNN(2, 2))
+    model.add(bb8.Reshape(-1, 7*7*64))
+
+    model.add(bb8.Linear(7*7*64, 100, with_bias=False))
+    model.add(bb8.BatchNormalization(100))
+    model.add(bb8.ReLU())
+    model.add(bb8.Dropout(0.5))
+
+    model.add(bb8.Linear(100, 10))
+    model.add(bb8.SoftMax())
+    return model
+

examples/MNIST/run.py

@@ -10,7 +10,7 @@ def main(params):
     train_set_x, train_set_y = train_set
     test_set_x, test_set_y = test_set
 
-    model = net()
+    model = lenet()
 
     criterion = bb8.ClassNLLCriterion()
 
@@ -19,13 +19,14 @@ def main(params):
     for epoch in range(100):
         model.training()
         train(train_set_x, train_set_y, model, optimiser, criterion, epoch, params['batch_size'])
+        train(train_set_x, train_set_y, model, optimiser, criterion, epoch, params['batch_size'], 'stat')
 
         model.evaluate()
         validate(test_set_x, test_set_y, model, epoch, params['batch_size'])
 
 
 if __name__ == "__main__":
     params = {}
-    params['lr'] = 0.001
+    params['lr'] = 0.1
     params['batch_size'] = 64
     main(params)

examples/MNIST/train.py

@@ -3,7 +3,7 @@
 import theano as _th
 
 
-def train(dataset_x, dataset_y, model, optimiser, criterion, epoch, batch_size):
+def train(dataset_x, dataset_y, model, optimiser, criterion, epoch, batch_size, mode=None):
     progress = make_progressbar('Training', epoch, len(dataset_x))
     progress.start()
 
@@ -17,9 +17,12 @@ def train(dataset_x, dataset_y, model, optimiser, criterion, epoch, batch_size):
             mini_batch_input[k] = dataset_x[shuffle[j * batch_size + k]]
             mini_batch_targets[k] = dataset_y[shuffle[j * batch_size + k]]
 
-        model.zero_grad_parameters()
-        model.accumulate_gradients(mini_batch_input, mini_batch_targets, criterion)
-        optimiser.update_parameters(model)
+        if mode is None:
+            model.zero_grad_parameters()
+            model.accumulate_gradients(mini_batch_input, mini_batch_targets, criterion)
+            optimiser.update_parameters(model)
+        else:
+            model.accumulate_statistics(mini_batch_input)
 
         progress.update(j * batch_size)