Merge branch 'master' into pow

Author: nudles

python/singa/autograd.py

@@ -1829,6 +1829,7 @@ def backward(self, dy):
 def leakyrelu(x, a=0.01):
     return LeakyRelu(a)(x)[0]
 
+  
 class Pow(Operation):
     def __init__(self):
         super(Pow, self).__init__()
@@ -1850,6 +1851,28 @@ def backward(self, dy):
 def pow(a, b):
     return Pow()(a,b)[0]
 
+class SoftSign(Operation):
+    def __init__(self):
+        super(SoftSign, self).__init__()  
+    
+    def forward(self, x):
+    # y = x / (1 + np.abs(x))
+        if training:
+            self.input = x
+        x1 = singa.AddFloat(singa.Abs(x),1.0)
+        y = singa.__div__(x,x1)
+        
+        return y
+      
+    def backward(self, dy):
+        dx = singa.AddFloat(singa.Abs(self.input),1.0)
+        dx = singa.PowFloat(singa.Square(dx),-1.0)
+        dx = singa.__mul__(dy, dx)
+        return dx
+      
+def softsign(x):
+    return SoftSign()(x)[0]
+
 class Sqrt(Operation):
     def __init__(self):
         super(Sqrt, self).__init__()  
@@ -1867,6 +1890,28 @@ def backward(self, dy):
 
 def sqrt(x):
     return Sqrt()(x)[0]
+  
+class SoftPlus(Operation):
+    def __init__(self):
+        super(SoftPlus, self).__init__()  
+    
+    def forward(self, x):
+    #f(x) = ln(exp(x) + 1)
+        if training:
+            self.input = x
+        x1 = singa.AddFloat(singa.Exp(x),1.0)
+        y = singa.Log(x1)    
+        return y
+
+    def backward(self, dy):
+        dx = singa.Exp(singa.MultFloat(self.input, -1.0))
+        dx = singa.PowFloat(singa.AddFloat(dx,1.0),-1.0)
+        dx = singa.__mul__(dy, dx)
+        return dx
+
+      
+def softplus(x):
+    return SoftPlus()(x)[0]
 
 class Sub(Operation):
     def __init__(self):

test/python/test_operation.py

@@ -817,7 +817,6 @@ def test_Sub_gpu(self):
 
         result = autograd.sub(x0, x1)
         dx0, dx1 = result.creator.backward(dy.data)
-
         DX0 = np.multiply(DY, 1.0)
         DX1 = np.multiply(DY, -1.0)
 
@@ -875,6 +874,86 @@ def test_Pow_gpu(self):
         np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx0)), DX0, decimal=4)
         np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx1)), DX1, decimal=4)
 
+    def test_SoftSign_cpu(self):
+        # y = x / (1 + np.abs(x))
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = X/(1 + np.absolute(X))
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(cpu_dev)
+        dy.to_device(cpu_dev)
+
+        result = autograd.softsign(x)
+        dx = result.creator.backward(dy.data)
+
+        G = 1.0/np.square(np.absolute(X)+1.0)
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+    
+    def test_SoftSign_gpu(self):
+        # y = x / (1 + np.abs(x))
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = X/(1 + np.absolute(X))
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(gpu_dev)
+        dy.to_device(gpu_dev)
+
+        result = autograd.softsign(x)
+        dx = result.creator.backward(dy.data)
+
+        G = 1.0/np.square(np.absolute(X)+1.0)
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+
+    def test_SoftPlus_cpu(self):
+        #y = np.log(np.exp(x) + 1)
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = np.log(np.exp(X) + 1)
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(cpu_dev)
+        dy.to_device(cpu_dev)
+
+        result = autograd.softplus(x)
+        dx = result.creator.backward(dy.data)
+        #dx = 1 / (1 + exp(-x))
+        G = 1.0 / (1.0 + np.exp(-X))
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+    
+    def test_SoftPlus_gpu(self):
+        #y = np.log(np.exp(x) + 1)
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = np.log(np.exp(X) + 1)
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(gpu_dev)
+        dy.to_device(gpu_dev)
+
+        result = autograd.softplus(x)
+        dx = result.creator.backward(dy.data)
+        #dx = 1 / (1 + exp(-x))
+        G = 1.0 / (1.0 + np.exp(-X))
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+    
     def test_Sqrt_cpu(self):
         X = np.array([0.1,1.0,0.4,4.0,0.9,9.0]).reshape(3,2).astype(np.float32)
         XT = np.sqrt(X)