Merge pull request #107 from lucasb-eyer/gelu

Add GELU non-linearity.

Author: lucasb-eyer

DeepFried2/layers/GELU.py

@@ -0,0 +1,9 @@
+import DeepFried2 as df
+
+class GELU(df.Module):
+    """ Gaussian Error Linear Unit (https://arxiv.org/abs/1606.08415) """
+
+    def symb_forward(self, x):
+        """ A very close, much more efficient approximation. """
+        return 0.5 * x * (1 + df.T.tanh(0.79788456 * (x + 0.044715 * x*x*x)))
+

DeepFried2/layers/__init__.py

@@ -3,6 +3,7 @@
 from .Bias import Bias
 from .Dropout import Dropout
 from .Embedding import Embedding
+from .GELU import GELU
 from .Identity import Identity
 from .Linear import Linear
 from .Log import Log