Merge pull request #5 from lucasb-eyer/moar-optimizers

Moar optimizers

Author: lucasb-eyer

DeepFried2/__init__.py

@@ -1,3 +1,4 @@
 from .layers import *
 from .containers import *
 from .criteria import *
+from .optimizers import *

DeepFried2/optimizers/AdaDelta.py

@@ -0,0 +1,55 @@
+from .Optimizer import Optimizer
+from ..utils import create_param_state_as
+
+from theano.tensor import sqrt
+
+
+class AdaDelta(Optimizer):
+    """
+    Implements Matt Zeiler's "Adaptive Learningrate" method, aka. AdaDelta.
+    The paper itself is really neat, and both very convincing and practical.
+
+    TL;DR: 1. AdaGrad quickly anneals, AdaDelta doesn't. (No proof.)
+           2. AdaGrad *is* sensitive to learning-rate, AdaGrad not so much. (Table 1.)
+           3. AdaGrad includes 2nd-order approximation. (3.2)
+
+    The updates are:
+
+        g²_{e+1} = ρ * g²_e + (1-ρ) * ∇p_e²
+        up_{e+1} = √(d²_e / g²_{e+1}) * ∇p_e
+        d²_{e+1} = ρ * d²_e + (1-ρ) * up²
+        p_{e+1} = p_e - up_{e+1}
+
+    As in RMSProp, we need to add epsilons in order to create stability.
+
+    It turns out that the effective learning-rate will converge to 1 as the
+    gradients decrease (and thus learning grinds to a halt). This could be used
+    to check for convergence by a specialized trainer.
+
+    The only reason `lr` is still there is this tweet by Alec Radford:
+
+    https://twitter.com/AlecRad/status/543518744799358977
+
+        @kastnerkyle @ogrisel @johnmyleswhite @tcovert Adadelta raw is finicky,
+        shrinking its updates by 0.5 "just works" in my experience as well.
+    """
+
+    def __init__(self, rho, eps=1e-7, lr=1):
+        Optimizer.__init__(self, rho=rho, eps=eps, lr=lr)
+
+    def get_updates(self, params, grads, rho, eps, lr):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            g2_state = create_param_state_as(param, prefix='g2_')
+            d2_state = create_param_state_as(param, prefix='d2_')
+
+            new_g2 = rho*g2_state + (1-rho)*grad*grad
+            up = lr * sqrt((d2_state+eps) / (new_g2+eps)) * grad
+            new_d2 = rho*d2_state + (1-rho)*up*up
+
+            updates.append((g2_state, new_g2))
+            updates.append((param, param - up))
+            updates.append((d2_state, new_d2))
+
+        return updates

DeepFried2/optimizers/AdaGrad.py

@@ -0,0 +1,47 @@
+from .Optimizer import Optimizer
+from ..utils import create_param_state_as
+
+from theano.tensor import sqrt
+
+
+class AdaGrad(Optimizer):
+    """
+    Implements Duchi's "Adaptive Subgradient" method, aka AdaGrad.
+    Chris Dyer's "Notes on AdaGrad" are pretty awesome for practical purposes.
+
+    TL;DR: AdaGrad doesn't need additional parameters (a lie) and makes the
+           optimization much less sensitive to the learning-rate!
+
+    In reality, it was a pioneer of fixing slow-learning features by adapting
+    a feature's own learning-rate using an estimate of its raw 2nd moment, but
+    its ideas have flown into superior AdaDelta and Adam.
+
+    The updates are:
+
+        g²_{e+1} = g²_e + ∇(p_e)²
+        p_{e+1} = p_e - (lr / √g²_{e+1}) * ∇p_e
+
+    that is, divide the learning-rate by a running square of the gradient.
+
+    Note that this would lead to division by 0 in the beginning for those
+    weights which don't receive a gradient (might be many with ReLUs), so we
+    initialize g² with a small value.
+    """
+
+    def __init__(self, lr, eps=1e-7):
+        Optimizer.__init__(self, lr=lr)
+
+        # eps is only needed as numeric value for initializing state and it's
+        # not possible to initialize state using symbolic variables.
+        self.eps=eps
+
+    def get_updates(self, params, grads, lr):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            g2_state = create_param_state_as(param, initial_value=self.eps)
+            new_g2 = g2_state + grad*grad
+            updates.append((g2_state, new_g2))
+            updates.append((param, param - lr/sqrt(new_g2) * grad))
+
+        return updates

DeepFried2/optimizers/Momentum.py

@@ -3,6 +3,17 @@
 
 
 class Momentum(Optimizer):
+    """
+    Implementation of the "Classical Momentum" (CM) which is explained in
+    further detail in
+
+    "On the importance of initialization and momentum in deep learning"
+
+    The updates are:
+
+        v_{e+1} = mom * v_e - lr * ∇p_e
+        p_{e+1} = p_e + v_{e+1}
+    """
 
     def __init__(self, lr, momentum):
         Optimizer.__init__(self, lr=lr, momentum=momentum)

DeepFried2/optimizers/Nesterov.py

@@ -0,0 +1,34 @@
+from .Optimizer import Optimizer
+from ..utils import create_param_state_as
+
+
+class Nesterov(Optimizer):
+    """
+    Implementation of "Nesterov's Accelerated Gradient" (NAG) which is explained
+    in further detail in
+
+    "On the importance of initialization and momentum in deep learning"
+
+    But the equation for NAG has been reshuffled by Nicolas Boulanger in
+
+    https://github.com/lisa-lab/pylearn2/pull/136#issuecomment-10381617
+
+    for easier implementation in Theano. The updates are:
+
+        v_{e+1} = mom * v_e - lr * ∇p_e
+        p_{e+1} = p_e + mom * v_{e+1} - lr * ∇p_e
+    """
+
+    def __init__(self, lr, momentum):
+        Optimizer.__init__(self, lr=lr, momentum=momentum)
+
+    def get_updates(self, params, grads, lr, momentum):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            param_mom = create_param_state_as(param)
+            v = momentum * param_mom - lr * grad
+            updates.append((param_mom, v))
+            updates.append((param, param + momentum * v - lr * grad))
+
+        return updates

DeepFried2/optimizers/Optimizer.py

@@ -24,3 +24,6 @@ def update_parameters(self, model):
 
     def get_updates(self, params, grads):
         raise NotImplementedError
+
+    def __repr__(self):
+        return type(self).__name__ + "(" + ", ".join(k+"="+str(v) for k,v in self.hyperparams.items()) + ")"

DeepFried2/optimizers/RMSProp.py

@@ -0,0 +1,35 @@
+from .Optimizer import Optimizer
+from ..utils import create_param_state_as
+
+from theano.tensor import sqrt
+
+
+class RMSProp(Optimizer):
+    """
+    Implements Hinton's "RMSProp" method presented in his Coursera lecture 6.5.
+    Essentially, it sits right in-between AdaGrad and AdaDelta by being a
+    windowed version of AdaGrad.
+
+    The updates are:
+
+        g²_{e+1} = ρ * g²_e + (1-ρ) * ∇p_e²
+        p_{e+1} = p_e - (lr / √g²_{e+1}) * ∇p_e
+
+    Note that in this case just initializing with epsilon is not enough anymore
+    as we might get zero-gradient for some units long enough to completely fill
+    the window.
+    """
+
+    def __init__(self, lr, rho, eps=1e-7):
+        Optimizer.__init__(self, lr=lr, rho=rho, eps=eps)
+
+    def get_updates(self, params, grads, lr, rho, eps):
+        updates = []
+
+        for param, grad in zip(params, grads):
+            g2_state = create_param_state_as(param)
+            new_g2 = rho*g2_state + (1-rho)*grad*grad
+            updates.append((g2_state, new_g2))
+            updates.append((param, param - lr/sqrt(new_g2+eps) * grad))
+
+        return updates

DeepFried2/optimizers/__init__.py

@@ -1,3 +1,7 @@
-from .Optimizer import *
-from .Momentum import *
-from .SGD import *
+from .Optimizer import Optimizer
+from .SGD import SGD
+from .Momentum import Momentum
+from .Nesterov import Nesterov
+from .AdaGrad import AdaGrad
+from .RMSProp import RMSProp
+from .AdaDelta import AdaDelta

DeepFried2/utils.py

@@ -14,8 +14,8 @@ def create_param_and_grad(shape, init, fan=None, name=None, type=_th.config.floa
     return param, grad_param
 
 
-def create_param_state_as(other, initial_value=0):
+def create_param_state_as(other, initial_value=0, prefix='state_for_'):
     return _th.shared(other.get_value()*0 + initial_value,
         broadcastable=other.broadcastable,
-        name='state_for_' + str(other.name)
+        name=prefix + str(other.name)
     )

examples/Optimizers/mnist.py

@@ -0,0 +1,29 @@
+import os
+import gzip
+import pickle
+import sys
+
+# Python 2/3 compatibility.
+try:
+    from urllib.request import urlretrieve
+except ImportError:
+    from urllib import urlretrieve
+
+
+'''Adapted from theano tutorial'''
+
+
+def load_mnist(data_file = os.path.join(os.path.dirname(__file__), 'mnist.pkl.gz')):
+
+    if not os.path.exists(data_file):
+        origin = 'http://www.iro.umontreal.ca/~lisa/deep/data/mnist/mnist.pkl.gz'
+        print('Downloading data from {}'.format(origin))
+        urlretrieve(origin, data_file)
+
+    print('... loading data')
+
+    with gzip.open(data_file, 'rb') as f:
+        if sys.version_info[0] == 3:
+            return pickle.load(f, encoding='latin1')
+        else:
+            return pickle.load(f)