Merge pull request #495 from pinpom/softsign

SINGA-475 add SoftSign operator

Author: nudles

python/singa/autograd.py

@@ -1830,6 +1830,28 @@ def leakyrelu(x, a=0.01):
     return LeakyRelu(a)(x)[0]
 
 
+class SoftSign(Operation):
+    def __init__(self):
+        super(SoftSign, self).__init__()  
+    
+    def forward(self, x):
+    # y = x / (1 + np.abs(x))
+        if training:
+            self.input = x
+        x1 = singa.AddFloat(singa.Abs(x),1.0)
+        y = singa.__div__(x,x1)
+        
+        return y
+      
+    def backward(self, dy):
+        dx = singa.AddFloat(singa.Abs(self.input),1.0)
+        dx = singa.PowFloat(singa.Square(dx),-1.0)
+        dx = singa.__mul__(dy, dx)
+        return dx
+      
+def softsign(x):
+    return SoftSign()(x)[0]
+
 class Sqrt(Operation):
     def __init__(self):
         super(Sqrt, self).__init__()  

test/python/test_operation.py

@@ -816,14 +816,53 @@ def test_Sub_gpu(self):
 
         result = autograd.sub(x0, x1)
         dx0, dx1 = result.creator.backward(dy.data)
-
         DX0 = np.multiply(DY, 1.0)
         DX1 = np.multiply(DY, -1.0)
 
         np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
         np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx0)), DX0, decimal=5)
         np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx1)), DX1, decimal=5)
 
+    def test_SoftSign_cpu(self):
+        # y = x / (1 + np.abs(x))
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = X/(1 + np.absolute(X))
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(cpu_dev)
+        dy.to_device(cpu_dev)
+
+        result = autograd.softsign(x)
+        dx = result.creator.backward(dy.data)
+
+        G = 1.0/np.square(np.absolute(X)+1.0)
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+    
+    def test_SoftSign_gpu(self):
+        # y = x / (1 + np.abs(x))
+        X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)
+        XT = X/(1 + np.absolute(X))
+        DY = np.ones((3, 2), dtype = np.float32)
+
+        x = tensor.from_numpy(X)
+        dy = tensor.from_numpy(DY)
+        x.to_device(gpu_dev)
+        dy.to_device(gpu_dev)
+
+        result = autograd.softsign(x)
+        dx = result.creator.backward(dy.data)
+
+        G = 1.0/np.square(np.absolute(X)+1.0)
+        DX = np.multiply(G, DY)
+
+        np.testing.assert_array_almost_equal(tensor.to_numpy(result), XT, decimal=5)
+        np.testing.assert_array_almost_equal(tensor.to_numpy(tensor.from_raw_tensor(dx)), DX, decimal=5)
+
     def test_SoftPlus_cpu(self):
         #y = np.log(np.exp(x) + 1)
         X = np.array([0.8, -1.2, 3.3, -3.6, -0.5, 0.5]).reshape(3, 2).astype(np.float32)