Merge pull request #106 from lucasb-eyer/crelu

Add CReLU layer.

Author: lucasb-eyer

DeepFried2/layers/ReLU.py

@@ -4,9 +4,18 @@
 
 class ReLU(df.Module):
 
-    def __init__(self, alpha = 0):
+    def __init__(self, alpha=0, caxis=None):
+        """ Fancy Rectified Linear Unit.
+        - `alpha` is the "leakyness", i.e. slope of negative part (0=relu, 1=linear).
+        - `caxis` can be specified to create a CReLU, [relu, -relu] along that axis.
+        """
         df.Module.__init__(self)
         self.alpha = alpha
+        self.caxis = caxis
 
-    def symb_forward(self, symb_input):
-        return relu(symb_input, self.alpha)
+    def symb_forward(self, x):
+        if self.caxis is None:
+            return relu(x, self.alpha)
+        else:
+            return df.T.concatenate([relu(x, self.alpha),
+                                     relu(-x, self.alpha)], axis=self.caxis)