Adds more common initializations.

Note this also fixes Xavier to correspond to real Xavier, it previously
didn't!

Author: lucasb-eyer

beacon8/init/Normal.py

@@ -0,0 +1,7 @@
+import numpy as _np
+
+
+def normal(std):
+    def init(shape, fan):
+        return std*_np.random.randn(*shape)
+    return init

beacon8/init/PReLU.py

@@ -0,0 +1,8 @@
+import numpy as _np
+from beacon8.init import xavier, xavierN
+
+def prelu(gain=1):
+    return xavier(gain * _np.sqrt(2))
+
+def preluN(gain=1):
+    return xavierN(gain * _np.sqrt(2))

beacon8/init/Uniform.py

@@ -0,0 +1,7 @@
+import numpy as _np
+
+
+def uniform(low, high):
+    def init(shape, fan):
+        return _np.random.uniform(low=low, high=high, size=shape)
+    return init

beacon8/init/Xavier.py

@@ -1,7 +1,24 @@
 import numpy as _np
 
-def xavier(shape, fan):
-    assert fan is not None, "The parameter's `fan` needs to be specified when using Xavier initialization."
+def xavier(gain=1):
+    def init(shape, fan):
+        assert fan is not None, "The parameter's `fan` needs to be specified when using Xavier initialization."
 
-    w_bound = _np.sqrt(4. / sum(fan))
-    return _np.random.uniform(low=-w_bound, high=w_bound, size=shape)
+        fan_mean = _np.mean(fan)
+        bound = _np.sqrt(3./fan_mean)
+        return _np.random.uniform(low=-bound, high=bound, size=shape)
+    return init
+
+def xavierN(gain=1):
+    def init(shape, fan):
+        assert fan is not None, "The parameter's `fan` needs to be specified when using Xavier initialization."
+
+        fan_mean = _np.mean(fan)
+        return _np.sqrt(1./fan_mean) * _np.random.randn(*shape)
+    return init
+
+def xavierSigm(gain=1):
+    return xavier(gain * _np.sqrt(2))
+
+def xavierSigmN(gain=1):
+    return xavierN(gain * _np.sqrt(2))

beacon8/init/__init__.py

@@ -1,2 +1,5 @@
 from .Const import const, zero, one
-from .Xavier import xavier
+from .Xavier import xavier, xavierN, xavierSigm, xavierSigmN
+from .PReLU import prelu, preluN
+from .Normal import normal
+from .Uniform import uniform

beacon8/layers/Linear.py

@@ -8,7 +8,7 @@
 
 class Linear(Module):
 
-    def __init__(self, nin, nout, init=xavier, with_bias=True, init_b=zero):
+    def __init__(self, nin, nout, init=xavier(), with_bias=True, init_b=zero):
         Module.__init__(self)
 
         self.nin = nin

beacon8/layers/SpatialConvolution.py

@@ -7,7 +7,7 @@
 
 
 class SpatialConvolution(Module):
-    def __init__(self, n_input_plane, n_output_plane, k_w, k_h, d_w=1, d_h=1, with_bias=True, border_mode='valid', imshape=None, init=xavier, init_b=zero):
+    def __init__(self, n_input_plane, n_output_plane, k_w, k_h, d_w=1, d_h=1, with_bias=True, border_mode='valid', imshape=None, init=xavier(), init_b=zero):
         Module.__init__(self)
         self.n_input_plane = n_input_plane
         self.n_output_plane = n_output_plane

beacon8/layers/SpatialConvolutionCUDNN.py

@@ -8,7 +8,7 @@
 
 
 class SpatialConvolutionCUDNN(Module):
-    def __init__(self, n_input_plane, n_output_plane, k_w, k_h, d_w=1, d_h=1, pad_w=0, pad_h=0, with_bias=True, init=xavier, init_b=zero):
+    def __init__(self, n_input_plane, n_output_plane, k_w, k_h, d_w=1, d_h=1, pad_w=0, pad_h=0, with_bias=True, init=xavier(), init_b=zero):
         Module.__init__(self)
         self.n_input_plane = n_input_plane
         self.n_output_plane = n_output_plane