Merge branch 'Add_load_ckpt' of github.com:Laicheng0830/tensorlayer into Add_load_ckpt

Author: Laicheng0830

docs/modules/activation.rst

@@ -35,6 +35,7 @@ For more complex activation, TensorFlow API will be required.
    sign
    hard_tanh
    pixel_wise_softmax
+   mish
 
 Ramp
 ------
@@ -68,6 +69,10 @@ Pixel-wise softmax
 --------------------
 .. autofunction:: pixel_wise_softmax
 
+mish
+---------
+.. autofunction:: mish
+
 Parametric activation
 ------------------------------
 See ``tensorlayer.layers``.

docs/user/contributing.rst

@@ -40,8 +40,10 @@ For TensorLayer 1.x, it was actively developed and maintained by the following p
 - **Hao Dong** (`@zsdonghao <https://github.com/zsdonghao>`_) - `<https://zsdonghao.github.io>`_
 - **Jonathan Dekhtiar** (`@DEKHTIARJonathan <https://github.com/DEKHTIARJonathan>`_) - `<https://www.jonathandekhtiar.eu>`_
 - **Luo Mai** (`@luomai <https://github.com/luomai>`_) - `<http://www.doc.ic.ac.uk/~lm111/>`_
+- **Pan Wang** (`@FerociousPanda <http://github.com/FerociousPanda>`_) - `<http://github.com/FerociousPanda>`_  (UI)
 - **Simiao Yu** (`@nebulaV <https://github.com/nebulaV>`_) - `<https://nebulav.github.io>`_
 
+
 Numerous other contributors can be found in the `Github Contribution Graph <https://github.com/tensorlayer/tensorlayer/graphs/contributors>`_.
 
 

tensorlayer/activation.py

@@ -19,6 +19,7 @@
     'htanh',
     'hard_tanh',
     'pixel_wise_softmax',
+    'mish',
 ]
 
 
@@ -339,6 +340,25 @@ def pixel_wise_softmax(x, name='pixel_wise_softmax'):
         return tf.nn.softmax(x)
 
 
+def mish(x):
+    """Mish activation function.
+
+    Reference: [Mish: A Self Regularized Non-Monotonic Neural Activation Function .Diganta Misra, 2019]<https://arxiv.org/abs/1908.08681>
+
+    Parameters
+    ----------
+    x : Tensor
+        input.
+
+    Returns
+    -------
+    Tensor
+        A ``Tensor`` in the same type as ``x``.
+
+    """
+    return x * tf.math.tanh(tf.math.softplus(x))
+
+
 # Alias
 lrelu = leaky_relu
 lrelu6 = leaky_relu6

tensorlayer/layers/normalization.py

@@ -107,6 +107,14 @@ def _bias_add(x, b, data_format):
         raise ValueError('invalid data_format: %s' % data_format)
 
 
+def _compute_shape(tensors):
+    if isinstance(tensors, list):
+        shape_mem = [t.get_shape().as_list() for t in tensors]
+    else:
+        shape_mem = tensors.get_shape().as_list()
+    return shape_mem
+
+
 def batch_normalization(x, mean, variance, offset, scale, variance_epsilon, data_format, name=None):
     """Data Format aware version of tf.nn.batch_normalization."""
     if data_format == 'channels_last':
@@ -256,7 +264,8 @@ def _get_param_shape(self, inputs_shape):
         return params_shape
 
     def _check_input_shape(self, inputs):
-        if inputs.ndim <= 1:
+        inputs_shape = _compute_shape(inputs)
+        if len(inputs_shape) <= 1:
             raise ValueError('expected input at least 2D, but got {}D input'.format(inputs.ndim))
 
     def build(self, inputs_shape):
@@ -318,7 +327,8 @@ class BatchNorm1d(BatchNorm):
     """
 
     def _check_input_shape(self, inputs):
-        if inputs.ndim != 2 and inputs.ndim != 3:
+        inputs_shape = _compute_shape(inputs)
+        if len(inputs_shape) != 2 and len(inputs_shape) != 3:
             raise ValueError('expected input to be 2D or 3D, but got {}D input'.format(inputs.ndim))
 
 
@@ -341,7 +351,8 @@ class BatchNorm2d(BatchNorm):
     """
 
     def _check_input_shape(self, inputs):
-        if inputs.ndim != 4:
+        inputs_shape = _compute_shape(inputs)
+        if len(inputs_shape) != 4:
             raise ValueError('expected input to be 4D, but got {}D input'.format(inputs.ndim))
 
 
@@ -364,7 +375,8 @@ class BatchNorm3d(BatchNorm):
     """
 
     def _check_input_shape(self, inputs):
-        if inputs.ndim != 5:
+        inputs_shape = _compute_shape(inputs)
+        if len(inputs_shape) != 5:
             raise ValueError('expected input to be 5D, but got {}D input'.format(inputs.ndim))
 
 

tests/test_activations.py

@@ -5,7 +5,7 @@
 import unittest
 
 import tensorflow as tf
-
+import numpy as np
 import tensorlayer as tl
 from tests.utils import CustomTestCase
 
@@ -116,6 +116,14 @@ def test_swish(self):
 
             self.assertAlmostEqual(computed_output.numpy(), good_output, places=5)
 
+    def test_mish(self):
+        for i in range(-5, 15):
+            good_output = i * np.tanh(np.math.log(1 + np.math.exp(i)))
+
+            computed_output = tl.act.mish(float(i))
+
+            self.assertAlmostEqual(computed_output.numpy(), good_output, places=5)
+
 
 if __name__ == '__main__':