[SYSTEMDS-3917] New built-in SELU activiation function

Closes #2328.

Author: e-strauss

scripts/nn/layers/selu.dml

@@ -0,0 +1,66 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+/*
+ * SeLU (Scaled Exponential Linear Unit) nonlinearity layer.
+ */
+
+forward = function(matrix[double] X)
+    return (matrix[double] out) {
+  /*
+   * Computes the forward pass for a SeLU nonlinearity layer.
+   *
+   *   selu(x) = lambda * ( x                if x > 0
+   *                 α * (exp(x) - 1)  if x <= 0 )
+   *
+   * Inputs:
+   *  - X: Inputs, of shape (any, any).
+   *
+   * Outputs:
+   *  - out: Outputs, of same shape as `X`.
+   */
+  alpha = 1.6732632423543772
+  lambda = 1.0507009873554805
+
+  out = (X > 0) * (lambda * X) +
+        (X <= 0) * (lambda * alpha * (exp(X) - 1))
+}
+
+backward = function(matrix[double] dout, matrix[double] X)
+    return (matrix[double] dX) {
+  /*
+   * Computes the backward pass for a SeLU nonlinearity layer.
+   *
+   * Inputs:
+   *  - dout: Gradient wrt `out` from upstream, of same shape as `X`.
+   *  - X: Inputs, of shape (any, any).
+   *
+   * Outputs:
+   *  - dX: Gradient wrt `X`, of same shape as `X`.
+   */
+  alpha = 1.6732632423543772
+  lambda = 1.0507009873554805
+
+  dselu = (X > 0) * lambda +
+          (X <= 0) * (lambda * alpha * exp(X))
+
+  dX = dselu * dout
+}

src/test/java/org/apache/sysds/test/applications/nn/NNComponentTest.java

@@ -129,6 +129,11 @@ public void gelu() {
 		run("gelu.dml");
 	}
 
+	@Test
+	public void selu() {
+		run("selu.dml");
+	}
+
 	@Test
 	public void embedding() {
 		run("embedding.dml");

src/test/scripts/applications/nn/component/selu.dml

@@ -0,0 +1,68 @@
+#-------------------------------------------------------------
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#-------------------------------------------------------------
+
+source("nn/layers/selu.dml") as selu
+source("src/test/scripts/applications/nn/util.dml") as test_util
+
+selu_test1 = function() {
+    print("Testing SeLU, test 1")
+
+    X = matrix("1.  -0.5
+                0.   2.", rows=2, cols=2)
+    dout = matrix("1 1
+                   1 1", rows=2, cols=2)
+
+    # Reference from PyTorch nn.SELU
+    out_expected = matrix("1.050701   -0.69175816
+                           0.         2.101402", rows=2, cols=2)
+    gradient_expected = matrix("1.050701   1.0663412
+                                1.7580993  1.050701", rows=2, cols=2)
+
+    out = selu::forward(X)
+    test_util::check_all_close(out, out_expected, 0.00001)
+
+    gradient = selu::backward(dout, X)
+    test_util::check_all_close(gradient, gradient_expected, 0.00001)
+}
+
+selu_test2 = function() {
+    print("Testing SeLU, test 2")
+
+    X = matrix("0.5  -1.5
+                1.   -2.", rows=2, cols=2)
+    dout = matrix("1 1
+                   1 1", rows=2, cols=2)
+
+    # Precomputed reference from PyTorch nn.SELU
+    out_expected = matrix("0.5253505   -1.3658143
+                           1.050701    -1.5201665", rows=2, cols=2)
+    gradient_expected = matrix("1.050701   0.392285
+                                1.050701   0.23793286", rows=2, cols=2)
+
+    out = selu::forward(X)
+    test_util::check_all_close(out, out_expected, 0.00001)
+
+    gradient = selu::backward(dout, X)
+    test_util::check_all_close(gradient, gradient_expected, 0.00001)
+}
+
+selu_test1()
+selu_test2()