Merge pull request #1 from lucasb-eyer/master

Initial commit to agree on architecture.

Author: ikostrikov2

README.md

@@ -1,2 +1,4 @@
 # Beacon8
 A Torch-inspired library for high-level deep learning with Theano.
+
+Thorough documentation will follow very soon.

beacon8/__init__.py

@@ -0,0 +1 @@
+from layer import *

beacon8/containers/__init__.py

@@ -0,0 +1,36 @@
+from .Layers import Module
+
+class Container(Module):
+
+    def __init__(self, *modules):
+        super().__init__()
+
+        self.modules = []
+        for module in modules:
+            self.add(module)
+
+    def evaluate(self):
+        super().evaluate()
+        for module in self.modules:
+            module.evaluate()
+
+    def training(self):
+        super().training()
+        for module in self.modules:
+            module.training()
+
+    def parameters(self):
+        params, grads = [], []
+
+        for module in self.modules:
+            mod_params, mod_grads = module.parameters()
+            params += mod_params
+            grads += mod_grads
+
+        return params, grads
+
+    def add(self, module):
+        self.modules.append(module)
+
+    def symbolic_forward(self, symbolic_input):
+        raise NotImplementedError

beacon8/layers/Linear.py

@@ -0,0 +1,37 @@
+from . import Module
+
+import numpy as _np
+import theano as _th
+
+class Linear(Module):
+
+    def __init__(self, nin, nout, init='Xavier', with_bias=True):
+        super().__init__()
+
+        self.nin = nin
+        self.nout = nout
+        self.init = init
+        self.with_bias = with_bias
+
+        self.reset()
+
+    def reset(self):
+        if self.init == 'Xavier':
+            w_bound = _np.sqrt(4 / (self.nin + self.nout))
+            W = _np.random.uniform(low=-w_bound, high=w_bound,
+                                   size=(self.nin, self.nout))
+        else:
+            raise NotImplementedError
+
+        self.weight = _th.shared(W.astype(_th.config.floatX))
+
+        if self.with_bias:
+            self.bias = _th.shared(_np.zeros(shape=self.nout, dtype=_th.config.floatX))
+
+    def symbolic_forward(self, symbolic_input):
+        out = _th.tensor.dot(symbolic_input, self.weight)
+
+        if self.with_bias:
+            out += self.bias
+
+        return out

beacon8/layers/Softmax.py

@@ -0,0 +1,11 @@
+from . import Module
+
+import theano.tensor as _T
+
+class SoftMax(Module):
+
+    def __init__(self):
+        Module.__init__(self)
+
+    def symbolic_forward(self, symbolic_input):
+        return _T.nnet.softmax(symbolic_input)

beacon8/layers/__init__.py

@@ -0,0 +1,72 @@
+import theano as _th
+import theano.tensor as _T
+import theano.config.floatX as _floatX
+
+class Module:
+
+    def __init__(self):
+        self.training_mode = True
+
+        self.fn_forward = None
+        self.fn_accum_grads = None
+
+    def reset(self):
+        pass
+
+    def __hash__(self):
+        raise NotImplementedError("You *need* to reimplement hash, even if it's just python's default. See the documentation for more info.")
+
+    def zero_grad_parameters(self):
+        _, grads = self.parameters()
+        for grad in grads:
+            grad.set_value(0 * grad.get_value())
+
+    def parameters(self):
+        params, grads = [], []
+
+        if self.training_mode and hasattr(self, 'weight'):
+            assert hasattr(self, 'grad_weight'), "The layer {} has a `weight` variable but no `grad_weight`, you probably forget to implement it.".format(type(self))
+            params += [self.weight]
+            grads += [self.grad_weight]
+
+        if self.training_mode and hasattr(self, 'bias'):
+            assert hasattr(self, 'grad_bias'), "The layer {} has a `bias` variable but no `grad_bias`, you probably forget to implement it.".format(type(self))
+            params += [self.bias]
+            grads += [self.grad_bias]
+
+        return params, grads
+
+    def evaluate(self):
+        self.training_mode = False
+
+    def training(self):
+        self.training_mode = True
+
+    def symb_forward(self, symb_input):
+        raise NotImplementedError
+
+    def forward(self, data):
+        if self.fn_forward is None:
+            symb_in = _T.TensorType(_floatX, (False,) * data.ndim)('X')
+            symb_out = self.symb_forward(symb_in)
+            self.fn_forward = _th.function(inputs=[symb_in], outputs=symb_out)
+
+        return self.fn_forward(data)
+
+    def accumulate_gradients(self, data_in, data_tgt, loss):
+        if self.fn_accum_grads is None:
+            symb_in = _T.TensorType(_floatX, (False,) * data_in.ndim)('X')
+            symb_tgt = _T.TensorType(_floatX, (False,) * data_tgt.ndim)('T')
+            symb_out = self.symbolic_forward(symb_in)
+            symb_err = loss.symb_forward(symb_out, symb_tgt)
+
+            params, grads = self.parameters()
+            symb_grads = theano.grad(cost=symb_err, wrt=params)
+
+            grads_updates = [(grad, grad + symb_grad) for grad, symb_grad in zip(grads, symb_grads)]
+            self.fn_accum_grads = _th.function(
+                inputs=[symb_in, symb_tgt],
+                updates=grads_updates
+            )
+
+        self.fn_accum_grads(data_in, data_tgt)

beacon8/optimizers/Momentum.py

@@ -0,0 +1,19 @@
+from .optimizers import Optimizer
+from .utils import create_param_state_as
+
+
+class Momentum(Optimizer):
+
+    def __init__(self, lr, momentum):
+        super().__init__(lr=lr, momentum=momentum)
+
+    def get_updates(self, params, grads, lr, momentum):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            param_mom = self.create_param_state_as(param)
+            v = momentum * param_mom - lr * grad
+            updates.append((param_mom, v))
+            updates.append((param, param + v))
+
+        return updates

beacon8/optimizers/SGD.py

@@ -0,0 +1,10 @@
+from .optimizers import Optimizer
+
+
+class SGD(Optimizer):
+
+    def __init__(self, lr):
+        super().__init__(lr=lr)
+
+    def get_updates(self, params, grads, lr):
+        return [g - lr * p for g, p in zip(grads, params)]

beacon8/optimizers/__init__.py

@@ -0,0 +1,25 @@
+import theano.tensor as _T
+
+
+class Optimizer:
+
+    def __init__(self, **hyperparams):
+        self.states = {}
+        self.hyperparams = hyperparams
+
+    def update_parameters(self, model):
+
+        if model not in self.states:
+            params, grads = model.parameters()
+            # TODO: Not only scalar
+            hyperparams = {name: _T.scalar(name) for name in self.hyperparams}
+            updates = self.get_updates(params, grads, **hyperparams)
+            self.states[model] = theano.function(
+                inputs=list(hyperparams.values()),
+                updates=updates
+            )
+
+        self.states[model](**self.hyperparams)
+
+    def get_updates(self, params, grads):
+        raise NotImplementedError

beacon8/utils.py

@@ -0,0 +1,7 @@
+import theano as _th
+
+def create_param_state_as(self, other, initial_value=0):
+    return _th.shared(other.get_value()*0 + initial_value,
+        broadcastable=other.broadcastable,
+        name='state_for_' + other.name
+    )