Work on Special Activations Functions (#690)

* work on special activations

* Special activation functions fixed

* Changelog modified

Author: Jonathan DEKHTIAR

CHANGELOG.md

@@ -179,6 +179,7 @@ To release a new version, please update the changelog as followed:
   - `tl.alphas` and `tl.alphas_like` added following the tf.ones/zeros and tf.zeros_like/ones_like (by @DEKHTIARJonathan in #580)
   - `tl.lazy_imports.LazyImport` to import heavy libraries only when necessary (by @DEKHTIARJonathan in #667)
   - `tl.act.leaky_relu6` and `tl.layers.PRelu6Layer` have been (by @DEKHTIARJonathan in #686)
+  - `tl.act.leaky_twice_relu6` and `tl.layers.PTRelu6Layer` have been (by @DEKHTIARJonathan in #690)
 - CI Tool:
   - [Stale Probot](https://github.com/probot/stale) added to clean stale issues (by @DEKHTIARJonathan in #573)
   - [Changelog Probot](https://github.com/mikz/probot-changelog) Configuration added (by @DEKHTIARJonathan in #637)
@@ -242,6 +243,7 @@ To release a new version, please update the changelog as followed:
 - Convolution Layers have been refactored under the folder `tensorlayer/layers/convolution` (by @DEKHTIARJonathan in #692)
 - Dense Layers have been refactored under the folder `tensorlayer/layers/dense` (by @DEKHTIARJonathan in #692)
 - Numerous Layer utility functions have been refactored in the file `tensorlayer/layers/utils.py` (by @DEKHTIARJonathan in #692)
+- `tests\test_layers_special_activation.py` made more robust for `PReluLayer`, `PRelu6Layer`, and `PTRelu6Layer` (by @DEKHTIARJonathan in #690)
 
 ### Deprecated
 - `tl.layers.TimeDistributedLayer` argurment `args` is deprecated in favor of `layer_args` (by @DEKHTIARJonathan in #667)

tensorlayer/activation.py

@@ -127,7 +127,7 @@ def leaky_relu6(x, alpha=0.2, name="leaky_relu6"):
     - `Convolutional Deep Belief Networks on CIFAR-10 [A. Krizhevsky, 2010] <http://www.cs.utoronto.ca/~kriz/conv-cifar10-aug2010.pdf>`__
     """
 
-    if not (0 < alpha <= 1):
+    if not isinstance(alpha, tf.Tensor) and not (0 < alpha <= 1):
         raise ValueError("`alpha` value must be in [0, 1]`")
 
     with tf.name_scope(name, "leaky_relu6") as name_scope:
@@ -179,10 +179,10 @@ def leaky_twice_relu6(x, alpha_low=0.2, alpha_high=0.2, name="leaky_relu6"):
 
     """
 
-    if not (0 < alpha_high <= 1):
+    if not isinstance(alpha_high, tf.Tensor) and not (0 < alpha_high <= 1):
         raise ValueError("`alpha_high` value must be in [0, 1]`")
 
-    if not (0 < alpha_low <= 1):
+    if not isinstance(alpha_low, tf.Tensor) and not (0 < alpha_low <= 1):
         raise ValueError("`alpha_low` value must be in [0, 1]`")
 
     with tf.name_scope(name, "leaky_twice_relu6") as name_scope:

tensorlayer/layers/core.py

@@ -281,8 +281,10 @@ def _add_dropout_layers(self, drop_layers):
             raise ValueError()
 
     @private_method
-    def _apply_activation(self, logits):
-        return self.act(logits) if self.act is not None else logits
+    def _apply_activation(self, logits, **kwargs):
+        if not kwargs:
+            kwargs = {}
+        return self.act(logits, **kwargs) if self.act is not None else logits
 
     @private_method
     def _argument_dict_checkup(self, args):

tensorlayer/layers/special_activation.py

@@ -50,7 +50,8 @@ def __init__(
             a_init_args=None, name="PReluLayer"
     ):
 
-        super(PReluLayer, self).__init__(prev_layer=prev_layer, a_init_args=a_init_args, name=name)
+        super(PReluLayer,
+              self).__init__(prev_layer=prev_layer, act=tf.nn.leaky_relu, a_init_args=a_init_args, name=name)
 
         if channel_shared:
             w_shape = (1, )
@@ -65,9 +66,14 @@ def __init__(
                 name='alpha', shape=w_shape, initializer=a_init, dtype=LayersConfig.tf_dtype, **self.a_init_args
             )
 
-            alpha_var = tf.nn.sigmoid(alpha_var, name="constraining_alpha_var_in_0_1")
+            alpha_var_constrained = tf.nn.sigmoid(alpha_var, name="constraining_alpha_var_in_0_1")
 
-        self.outputs = tf.nn.leaky_relu(self.inputs, alpha=alpha_var, name="PReLU_activation")
+        self.outputs = self._apply_activation(
+            self.inputs, **{
+                'alpha': alpha_var_constrained,
+                'name': "PReLU_activation"
+            }
+        )
 
         self._add_layers(self.outputs)
         self._add_params(alpha_var)
@@ -119,7 +125,7 @@ def __init__(
             a_init_args=None, name="PReLU6_layer"
     ):
 
-        super(PRelu6Layer, self).__init__(prev_layer=prev_layer, a_init_args=a_init_args, name=name)
+        super(PRelu6Layer, self).__init__(prev_layer=prev_layer, act=leaky_relu6, a_init_args=a_init_args, name=name)
 
         if channel_shared:
             w_shape = (1, )
@@ -134,9 +140,14 @@ def __init__(
                 name='alpha', shape=w_shape, initializer=a_init, dtype=LayersConfig.tf_dtype, **self.a_init_args
             )
 
-            alpha_var = tf.nn.sigmoid(alpha_var, name="constraining_alpha_var_in_0_1")
+            alpha_var_constrained = tf.nn.sigmoid(alpha_var, name="constraining_alpha_var_in_0_1")
 
-        self.outputs = leaky_relu6(self.inputs, alpha=alpha_var, name="PReLU6_activation")
+        self.outputs = self._apply_activation(
+            self.inputs, **{
+                'alpha': alpha_var_constrained,
+                'name': "PReLU6_activation"
+            }
+        )
 
         self._add_layers(self.outputs)
         self._add_params(alpha_var)
@@ -190,7 +201,8 @@ def __init__(
             a_init_args=None, name="PTReLU6_layer"
     ):
 
-        super(PTRelu6Layer, self).__init__(prev_layer=prev_layer, a_init_args=a_init_args, name=name)
+        super(PTRelu6Layer,
+              self).__init__(prev_layer=prev_layer, act=leaky_twice_relu6, a_init_args=a_init_args, name=name)
 
         if channel_shared:
             w_shape = (1, )
@@ -207,17 +219,21 @@ def __init__(
                 name='alpha_low', shape=w_shape, initializer=a_init, dtype=LayersConfig.tf_dtype, **self.a_init_args
             )
 
-            alpha_low = tf.nn.sigmoid(alpha_low, name="constraining_alpha_low_in_0_1")
+            alpha_low_constrained = tf.nn.sigmoid(alpha_low, name="constraining_alpha_low_in_0_1")
 
             # Alpha for outputs higher than 6
             alpha_high = tf.get_variable(
                 name='alpha_high', shape=w_shape, initializer=a_init, dtype=LayersConfig.tf_dtype, **self.a_init_args
             )
 
-            alpha_high = tf.nn.sigmoid(alpha_high, name="constraining_alpha_high_in_0_1")
+            alpha_high_constrained = tf.nn.sigmoid(alpha_high, name="constraining_alpha_high_in_0_1")
 
-        self.outputs = leaky_twice_relu6(
-            self.inputs, alpha_low=alpha_low, alpha_high=alpha_high, name="PTReLU6_activation"
+        self.outputs = self.outputs = self._apply_activation(
+            self.inputs, **{
+                'alpha_low': alpha_low_constrained,
+                'alpha_high': alpha_high_constrained,
+                'name': "PTReLU6_activation"
+            }
         )
 
         self._add_layers(self.outputs)

tests/test_layers_special_activation.py

@@ -6,49 +6,136 @@
 import tensorlayer as tl
 
 
-class Layer_Special_Activation_Test(unittest.TestCase):
+class PReLU_Layer_Test(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
 
         x = tf.placeholder(tf.float32, shape=[None, 30])
-        net = tl.layers.InputLayer(x, name='input')
-        net = tl.layers.DenseLayer(net, n_units=10, name='dense')
-        net1 = tl.layers.PReluLayer(net, name='prelu')
 
-        net1.print_layers()
-        net1.print_params(False)
+        in_layer = tl.layers.InputLayer(x, name='input')
 
-        cls.net1_shape = net1.outputs.get_shape().as_list()
-        cls.net1_layers = net1.all_layers
-        cls.net1_params = net1.all_params
-        cls.net1_n_params = net1.count_params()
+        net = tl.layers.DenseLayer(in_layer, n_units=10, name='dense_1')
+        cls.net1 = tl.layers.PReluLayer(net, name='prelu_1')
 
-        net2 = tl.layers.PReluLayer(net1, channel_shared=True, name='prelu2')
+        cls.net1.print_layers()
+        cls.net1.print_params(False)
 
-        net2.print_layers()
-        net2.print_params(False)
+        net2 = tl.layers.DenseLayer(cls.net1, n_units=30, name='dense_2')
+        cls.net2 = tl.layers.PReluLayer(net2, channel_shared=True, name='prelu_2')
 
-        cls.net2_shape = net2.outputs.get_shape().as_list()
-        cls.net2_layers = net2.all_layers
-        cls.net2_params = net2.all_params
-        cls.net2_n_params = net2.count_params()
+        cls.net2.print_layers()
+        cls.net2.print_params(False)
 
     @classmethod
     def tearDownClass(cls):
         tf.reset_default_graph()
 
     def test_net1(self):
-        self.assertEqual(len(self.net1_layers), 3)
-        self.assertEqual(len(self.net1_params), 3)
-        self.assertEqual(self.net1_n_params, 320)
-        self.assertEqual(self.net1_shape[-1], 10)
+        self.assertEqual(len(self.net1.all_layers), 3)
+        self.assertEqual(len(self.net1.all_params), 3)
+        self.assertEqual(self.net1.count_params(), 320)
+        self.assertEqual(self.net1.outputs.get_shape().as_list()[1:], [10])
+
+        prelu1_param_shape = self.net1.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu1_param_shape, [10])
+
+    def test_net2(self):
+        self.assertEqual(len(self.net2.all_layers), 5)
+        self.assertEqual(len(self.net2.all_params), 6)
+        self.assertEqual(self.net2.count_params(), 651)
+        self.assertEqual(self.net2.outputs.get_shape().as_list()[1:], [30])
+
+        prelu2_param_shape = self.net2.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu2_param_shape, [1])
+
+
+class PRelu6_Layer_Test(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+
+        x = tf.placeholder(tf.float32, shape=[None, 30])
+
+        in_layer = tl.layers.InputLayer(x, name='input')
+
+        net = tl.layers.DenseLayer(in_layer, n_units=10, name='dense_1')
+        cls.net1 = tl.layers.PRelu6Layer(net, name='prelu6_1')
+
+        cls.net1.print_layers()
+        cls.net1.print_params(False)
+
+        net2 = tl.layers.DenseLayer(cls.net1, n_units=30, name='dense_2')
+        cls.net2 = tl.layers.PRelu6Layer(net2, channel_shared=True, name='prelu6_2')
+
+        cls.net2.print_layers()
+        cls.net2.print_params(False)
+
+    @classmethod
+    def tearDownClass(cls):
+        tf.reset_default_graph()
+
+    def test_net1(self):
+        self.assertEqual(len(self.net1.all_layers), 3)
+        self.assertEqual(len(self.net1.all_params), 3)
+        self.assertEqual(self.net1.count_params(), 320)
+        self.assertEqual(self.net1.outputs.get_shape().as_list()[1:], [10])
+
+        prelu1_param_shape = self.net1.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu1_param_shape, [10])
 
     def test_net2(self):
-        self.assertEqual(len(self.net2_layers), 4)
-        self.assertEqual(len(self.net2_params), 4)
-        self.assertEqual(self.net2_n_params, 321)
-        self.assertEqual(self.net2_shape[-1], 10)
+        self.assertEqual(len(self.net2.all_layers), 5)
+        self.assertEqual(len(self.net2.all_params), 6)
+        self.assertEqual(self.net2.count_params(), 651)
+        self.assertEqual(self.net2.outputs.get_shape().as_list()[1:], [30])
+
+        prelu2_param_shape = self.net2.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu2_param_shape, [1])
+
+
+class PTRelu6_Layer_Test(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls):
+
+        x = tf.placeholder(tf.float32, shape=[None, 30])
+
+        in_layer = tl.layers.InputLayer(x, name='input')
+
+        net = tl.layers.DenseLayer(in_layer, n_units=10, name='dense_1')
+        cls.net1 = tl.layers.PTRelu6Layer(net, name='ptrelu6_1')
+
+        cls.net1.print_layers()
+        cls.net1.print_params(False)
+
+        net2 = tl.layers.DenseLayer(cls.net1, n_units=30, name='dense_2')
+        cls.net2 = tl.layers.PTRelu6Layer(net2, channel_shared=True, name='ptrelu6_2')
+
+        cls.net2.print_layers()
+        cls.net2.print_params(False)
+
+    @classmethod
+    def tearDownClass(cls):
+        tf.reset_default_graph()
+
+    def test_net1(self):
+        self.assertEqual(len(self.net1.all_layers), 3)
+        self.assertEqual(len(self.net1.all_params), 4)
+        self.assertEqual(self.net1.count_params(), 330)
+        self.assertEqual(self.net1.outputs.get_shape().as_list()[1:], [10])
+
+        prelu1_param_shape = self.net1.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu1_param_shape, [10])
+
+    def test_net2(self):
+        self.assertEqual(len(self.net2.all_layers), 5)
+        self.assertEqual(len(self.net2.all_params), 8)
+        self.assertEqual(self.net2.count_params(), 662)
+        self.assertEqual(self.net2.outputs.get_shape().as_list()[1:], [30])
+
+        prelu2_param_shape = self.net2.all_params[-1].get_shape().as_list()
+        self.assertEqual(prelu2_param_shape, [1])
 
 
 if __name__ == '__main__':