Added a MNIST example. Fixed typos.

Compatibility fixes.

Compatibility fix 2.

Author: ikostrikov2

beacon8/__init__.py

@@ -1 +1,3 @@
-from layer import *
+from .layers import *
+from .containers import *
+from .criteria import *

beacon8/containers/Container.py

@@ -0,0 +1,34 @@
+from ..layers import Module
+
+
+class Container(Module):
+
+    def __init__(self, *modules):
+        Module.__init__(self)
+
+        self.modules = []
+        for module in modules:
+            self.add(module)
+
+    def evaluate(self):
+        Module.evaluate(self)
+        for module in self.modules:
+            module.evaluate()
+
+    def training(self):
+        Module.training(self)
+        for module in self.modules:
+            module.training()
+
+    def parameters(self):
+        params, grads = [], []
+
+        for module in self.modules:
+            mod_params, mod_grads = module.parameters()
+            params += mod_params
+            grads += mod_grads
+
+        return params, grads
+
+    def add(self, module):
+        self.modules.append(module)

beacon8/containers/Sequential.py

@@ -0,0 +1,9 @@
+from .Container import Container
+
+
+class Sequential(Container):
+    def symb_forward(self, symb_input):
+        symb_output = symb_input
+        for module in self.modules:
+            symb_output = module.symb_forward(symb_output)
+        return symb_output

beacon8/containers/__init__.py

@@ -1,36 +1,2 @@
-from .Layers import Module
-
-class Container(Module):
-
-    def __init__(self, *modules):
-        super().__init__()
-
-        self.modules = []
-        for module in modules:
-            self.add(module)
-
-    def evaluate(self):
-        super().evaluate()
-        for module in self.modules:
-            module.evaluate()
-
-    def training(self):
-        super().training()
-        for module in self.modules:
-            module.training()
-
-    def parameters(self):
-        params, grads = [], []
-
-        for module in self.modules:
-            mod_params, mod_grads = module.parameters()
-            params += mod_params
-            grads += mod_grads
-
-        return params, grads
-
-    def add(self, module):
-        self.modules.append(module)
-
-    def symbolic_forward(self, symbolic_input):
-        raise NotImplementedError
+from .Container import *
+from .Sequential import *

beacon8/criteria/ClassNLLCriterion.py

@@ -0,0 +1,7 @@
+import theano.tensor as _T
+
+
+class ClassNLLCriterion:
+    def symb_forward(self, symb_input, symb_targets):
+        int_targets = _T.cast(symb_targets, 'int32')
+        return _T.mean(-_T.log(symb_input[_T.arange(symb_targets.shape[0]), int_targets]))

beacon8/criteria/__init__.py

@@ -0,0 +1 @@
+from .ClassNLLCriterion import *

beacon8/layers/Linear.py

@@ -1,12 +1,13 @@
-from . import Module
+from .Module import Module
 
 import numpy as _np
 import theano as _th
 
+
 class Linear(Module):
 
     def __init__(self, nin, nout, init='Xavier', with_bias=True):
-        super().__init__()
+        Module.__init__(self)
 
         self.nin = nin
         self.nout = nout
@@ -17,19 +18,21 @@ def __init__(self, nin, nout, init='Xavier', with_bias=True):
 
     def reset(self):
         if self.init == 'Xavier':
-            w_bound = _np.sqrt(4 / (self.nin + self.nout))
+            w_bound = _np.sqrt(4. / (self.nin + self.nout))
             W = _np.random.uniform(low=-w_bound, high=w_bound,
                                    size=(self.nin, self.nout))
         else:
             raise NotImplementedError
 
         self.weight = _th.shared(W.astype(_th.config.floatX))
+        self.grad_weight = _th.shared((W*0.).astype(_th.config.floatX))
 
         if self.with_bias:
             self.bias = _th.shared(_np.zeros(shape=self.nout, dtype=_th.config.floatX))
+            self.grad_bias = _th.shared(_np.zeros(shape=self.nout, dtype=_th.config.floatX))
 
-    def symbolic_forward(self, symbolic_input):
-        out = _th.tensor.dot(symbolic_input, self.weight)
+    def symb_forward(self, symb_input):
+        out = _th.tensor.dot(symb_input, self.weight)
 
         if self.with_bias:
             out += self.bias

beacon8/layers/Module.py

@@ -0,0 +1,72 @@
+import theano as _th
+import theano.tensor as _T
+
+
+class Module:
+
+    def __init__(self):
+        self.training_mode = True
+
+        self.fn_forward = None
+        self.fn_accum_grads = None
+
+    def reset(self):
+        pass
+
+    #def __hash__(self):
+    #    raise NotImplementedError("You *need* to reimplement hash, even if it's just python's default. See the documentation for more info.")
+
+    def zero_grad_parameters(self):
+        _, grads = self.parameters()
+        for grad in grads:
+            grad.set_value(0 * grad.get_value())
+
+    def parameters(self):
+        params, grads = [], []
+
+        if self.training_mode and hasattr(self, 'weight'):
+            assert hasattr(self, 'grad_weight'), "The layer {} has a `weight` variable but no `grad_weight`, you probably forget to implement it.".format(type(self))
+            params += [self.weight]
+            grads += [self.grad_weight]
+
+        if self.training_mode and hasattr(self, 'bias'):
+            assert hasattr(self, 'grad_bias'), "The layer {} has a `bias` variable but no `grad_bias`, you probably forget to implement it.".format(type(self))
+            params += [self.bias]
+            grads += [self.grad_bias]
+
+        return params, grads
+
+    def evaluate(self):
+        self.training_mode = False
+
+    def training(self):
+        self.training_mode = True
+
+    def symb_forward(self, symb_input):
+        raise NotImplementedError
+
+    def forward(self, data):
+        if self.fn_forward is None:
+            symb_in = _T.TensorType(_th.config.floatX, (False,) * data.ndim)('X')
+            symb_out = self.symb_forward(symb_in)
+            self.fn_forward = _th.function(inputs=[symb_in], outputs=symb_out)
+
+        return self.fn_forward(data)
+
+    def accumulate_gradients(self, data_in, data_tgt, loss):
+        if self.fn_accum_grads is None:
+            symb_in = _T.TensorType(_th.config.floatX, (False,) * data_in.ndim)('X')
+            symb_tgt = _T.TensorType(_th.config.floatX, (False,) * data_tgt.ndim)('T')
+            symb_out = self.symb_forward(symb_in)
+            symb_err = loss.symb_forward(symb_out, symb_tgt)
+
+            params, grads = self.parameters()
+            symb_grads = _th.grad(cost=symb_err, wrt=params)
+
+            grads_updates = [(grad, grad + symb_grad) for grad, symb_grad in zip(grads, symb_grads)]
+            self.fn_accum_grads = _th.function(
+                inputs=[symb_in, symb_tgt],
+                updates=grads_updates
+            )
+
+        self.fn_accum_grads(data_in, data_tgt)

beacon8/layers/ReLU.py

@@ -0,0 +1,7 @@
+from .Module import Module
+
+
+class ReLU(Module):
+
+    def symb_forward(self, symb_input):
+        return (symb_input + abs(symb_input)) * 0.5

beacon8/layers/Softmax.py

@@ -2,10 +2,11 @@
 
 import theano.tensor as _T
 
+
 class SoftMax(Module):
 
     def __init__(self):
         Module.__init__(self)
 
-    def symbolic_forward(self, symbolic_input):
-        return _T.nnet.softmax(symbolic_input)
+    def symb_forward(self, symb_input):
+        return _T.nnet.softmax(symb_input)