Merge pull request #7535 from guoshengCS/add-glu-act

Add activation for glu

Author: guoshengCS

python/paddle/v2/fluid/nets.py

@@ -106,9 +106,10 @@ def sequence_conv_pool(input,
 
 def glu(input, dim=-1):
     """
-    The gated linear unit composed by split and elementwise multiplication. 
-    Specifically, Split the input into two equal sized parts :math:`a` and 
-    :math:`b` along the given dimension and then compute as following:
+    The gated linear unit composed by split, sigmoid activation and elementwise 
+    multiplication. Specifically, Split the input into two equal sized parts 
+    :math:`a` and :math:`b` along the given dimension and then compute as 
+    following:
 
         .. math::
 
@@ -133,5 +134,6 @@ def glu(input, dim=-1):
     """
 
     a, b = layers.split(input, num_or_sections=2, dim=dim)
-    out = layers.elementwise_mul(x=a, y=b)
+    act_b = layers.sigmoid(x=b)
+    out = layers.elementwise_mul(x=a, y=act_b)
     return out