Merge pull request #1068 from Laicheng0830/ADD_Mish

Add mish

Author: Laicheng0830

docs/modules/activation.rst

@@ -35,6 +35,7 @@ For more complex activation, TensorFlow API will be required.
    sign
    hard_tanh
    pixel_wise_softmax
+   mish
 
 Ramp
 ------
@@ -68,6 +69,10 @@ Pixel-wise softmax
 --------------------
 .. autofunction:: pixel_wise_softmax
 
+mish
+---------
+.. autofunction:: mish
+
 Parametric activation
 ------------------------------
 See ``tensorlayer.layers``.

tensorlayer/activation.py

@@ -19,6 +19,7 @@
     'htanh',
     'hard_tanh',
     'pixel_wise_softmax',
+    'mish',
 ]
 
 
@@ -339,6 +340,25 @@ def pixel_wise_softmax(x, name='pixel_wise_softmax'):
         return tf.nn.softmax(x)
 
 
+def mish(x):
+    """Mish activation function.
+
+    Reference: [Mish: A Self Regularized Non-Monotonic Neural Activation Function .Diganta Misra, 2019]<https://arxiv.org/abs/1908.08681>
+
+    Parameters
+    ----------
+    x : Tensor
+        input.
+
+    Returns
+    -------
+    Tensor
+        A ``Tensor`` in the same type as ``x``.
+
+    """
+    return x * tf.math.tanh(tf.math.softplus(x))
+
+
 # Alias
 lrelu = leaky_relu
 lrelu6 = leaky_relu6

tests/test_activations.py

@@ -5,7 +5,7 @@
 import unittest
 
 import tensorflow as tf
-
+import numpy as np
 import tensorlayer as tl
 from tests.utils import CustomTestCase
 
@@ -116,6 +116,14 @@ def test_swish(self):
 
             self.assertAlmostEqual(computed_output.numpy(), good_output, places=5)
 
+    def test_mish(self):
+        for i in range(-5, 15):
+            good_output = i * np.tanh(np.math.log(1 + np.math.exp(i)))
+
+            computed_output = tl.act.mish(float(i))
+
+            self.assertAlmostEqual(computed_output.numpy(), good_output, places=5)
+
 
 if __name__ == '__main__':