Merge pull request #377 from apphp/SAM-10-SELU

Sam 10 selu and sigmoid

Author: SkibidiProduction

docs/neural-network/activation-functions/elu.md

@@ -32,4 +32,4 @@ $activationFunction = new ELU(2.5);
 ```
 
 ## References
-[^1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.
+[1]: D. A. Clevert et al. (2016). Fast and Accurate Deep Network Learning by Exponential Linear Units.

docs/neural-network/activation-functions/hard-sigmoid.md

@@ -26,4 +26,4 @@ $activationFunction = new HardSigmoid();
 ```
 
 ## References
-[^1]: https://en.wikipedia.org/wiki/Hard_sigmoid
+[1]: https://en.wikipedia.org/wiki/Hard_sigmoid

docs/neural-network/activation-functions/leaky-relu.md

@@ -32,4 +32,4 @@ $activationFunction = new LeakyReLU(0.3);
 ```
 
 ## References
-[^1]: A. L. Maas et al. (2013). Rectifier Nonlinearities Improve Neural Network Acoustic Models.
+[1]: A. L. Maas et al. (2013). Rectifier Nonlinearities Improve Neural Network Acoustic Models.

docs/neural-network/activation-functions/relu.md

@@ -30,5 +30,5 @@ $activationFunction = new ReLU(0.1);
 ```
 
 ## References
-[^1]: A. L. Maas et al. (2013). Rectifier Nonlinearities Improve Neural Network Acoustic Models.
-[^2]: K. Konda et al. (2015). Zero-bias Autoencoders and the Benefits of Co-adapting Features.
+[1]: A. L. Maas et al. (2013). Rectifier Nonlinearities Improve Neural Network Acoustic Models.
+[2]: K. Konda et al. (2015). Zero-bias Autoencoders and the Benefits of Co-adapting Features.

docs/neural-network/activation-functions/selu.md

@@ -34,4 +34,4 @@ $activationFunction = new SELU();
 ```
 
 ## References
-[^1]: G. Klambauer et al. (2017). Self-Normalizing Neural Networks.
+[1]: G. Klambauer et al. (2017). Self-Normalizing Neural Networks.

docs/neural-network/activation-functions/silu.md

@@ -28,5 +28,5 @@ use Rubix\ML\NeuralNet\ActivationFunctions\SiLU;
 $activationFunction = new SiLU();
 ```
 
-### References
-[^1]: S. Elwing et al. (2017). Sigmoid-Weighted Linear Units for Neural Network Function Approximation in Reinforcement Learning.
+## References
+[1]: S. Elwing et al. (2017). Sigmoid-Weighted Linear Units for Neural Network Function Approximation in Reinforcement Learning.

docs/neural-network/activation-functions/softplus.md

@@ -26,4 +26,4 @@ $activationFunction = new Softplus();
 ```
 
 ## References
-[^1]: X. Glorot et al. (2011). Deep Sparse Rectifier Neural Networks.
+[1]: X. Glorot et al. (2011). Deep Sparse Rectifier Neural Networks.

docs/neural-network/activation-functions/softsign.md

@@ -26,4 +26,4 @@ $activationFunction = new Softsign();
 ```
 
 ## References
-[^1]: X. Glorot et al. (2010). Understanding the Difficulty of Training Deep Feedforward Neural Networks.
+[1]: X. Glorot et al. (2010). Understanding the Difficulty of Training Deep Feedforward Neural Networks.

docs/neural-network/activation-functions/thresholded-relu.md

@@ -32,4 +32,4 @@ $activationFunction = new ThresholdedReLU(2.0);
 ```
 
 ## References
-[^1]: K. Konda et al. (2015). Zero-bias autoencoders and the benefits of co-adapting features.
+[1]: K. Konda et al. (2015). Zero-bias autoencoders and the benefits of co-adapting features.

src/NeuralNet/ActivationFunctions/SELU/SELU.php

@@ -1,8 +1,13 @@
 <?php
 
+declare(strict_types=1);
+
 namespace Rubix\ML\NeuralNet\ActivationFunctions\SELU;
 
-use Tensor\Matrix;
+use NumPower;
+use NDArray;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\ActivationFunction;
+use Rubix\ML\NeuralNet\ActivationFunctions\Base\Contracts\IBufferDerivative;
 
 /**
  * SELU
@@ -18,69 +23,94 @@
  * @category    Machine Learning
  * @package     Rubix/ML
  * @author      Andrew DalPino
+ * @author      Samuel Akopyan <leumas.a@gmail.com>
  */
-class SELU implements ActivationFunction
+class SELU implements ActivationFunction, IBufferDerivative
 {
     /**
      * The value at which leakage starts to saturate.
      *
      * @var float
      */
-    public const ALPHA = 1.6732632423543772848170429916717;
+    public const ALPHA = 1.6732632;
 
     /**
      * The scaling coefficient.
      *
      * @var float
      */
-    public const SCALE = 1.0507009873554804934193349852946;
+    public const LAMBDA = 1.0507009;
 
     /**
      * The scaling coefficient multiplied by alpha.
      *
      * @var float
      */
-    protected const BETA = self::SCALE * self::ALPHA;
+    protected const BETA = self::LAMBDA * self::ALPHA;
 
     /**
      * Compute the activation.
      *
-     * @internal
+     * f(x) = λ * x                 if x > 0
+     * f(x) = λ * α * (e^x - 1)     if x ≤ 0
      *
-     * @param Matrix $input
-     * @return Matrix
+     * @param NDArray $input The input values
+     * @return NDArray The activated values
      */
-    public function activate(Matrix $input) : Matrix
+    public function activate(NDArray $input) : NDArray
     {
-        $positive = NumPower::maximum($input, 0) * self::SCALE;
-        $negative = self::BETA * NumPower::expm1($input);
+        // Calculate positive part: λ * x for x > 0
+        $positive = NumPower::multiply(
+            self::LAMBDA,
+            NumPower::maximum($input, 0)
+        );
+
+        // Calculate negative part: λ * α * (e^x - 1) for x <= 0
+        $negativeMask = NumPower::minimum($input, 0);
+        $negative = NumPower::multiply(
+            self::BETA,
+            NumPower::expm1($negativeMask)
+        );
 
-        return $negative + $positive;
+        // Combine both parts
+        return NumPower::add($positive, $negative);
     }
 
     /**
-     * Calculate the derivative of the activation.
+     * Calculate the derivative of the SELU activation function.
      *
-     * @internal
+     * f'(x) = λ                if x > 0
+     * f'(x) = λ * α * e^x      if x ≤ 0
      *
-     * @param Matrix $input
-     * @param Matrix $output
-     * @return Matrix
+     * @param NDArray $input Input matrix
+     * @return NDArray Derivative matrix
      */
-    public function differentiate(Matrix $input, Matrix $output) : Matrix
+    public function differentiate(NDArray $input) : NDArray
     {
-        $positive = NumPower::greater($output, 0) * self::SCALE;
-        $negative = NumPower::lessEqual($output) * ($output + self::ALPHA) * self::SCALE;
+        // For x > 0: λ
+        $positiveMask = NumPower::greater($input, 0);
+        $positivePart = NumPower::multiply($positiveMask, self::LAMBDA);
 
-        return $positive + $negative;
+        // For x <= 0: λ * α * e^x
+        $negativeMask = NumPower::lessEqual($input, 0);
+        $negativePart = NumPower::multiply(
+            NumPower::multiply(
+                NumPower::exp(
+                    NumPower::multiply($negativeMask, $input)
+                ),
+                self::BETA
+            ),
+            $negativeMask
+        );
+
+        // Combine both parts
+        return NumPower::add($positivePart, $negativePart);
     }
 
     /**
-     * Return the string representation of the object.
-     *
-     * @internal
+     * Return the string representation of the activation function.
      *
-     * @return string
+     * @return string String representation
      */
     public function __toString() : string
     {