add leaky relu activation

Author: jalvesz

doc/specs/stdlib_specialfunctions_activations.md

@@ -203,6 +203,79 @@ Elemental function
 
 The function returns a value with the same type and kind as input argument.
 
+## `Leaky_relu` - Leaky Rectified Linear Unit function
+
+### Status
+
+Experimental
+
+### Description
+
+Computes the gaussian function:
+$$
+\text{f}(x) =
+\begin{cases} 
+x, & \text{if } x \geq 0 \\
+a * x, & \text{otherwise}
+\end{cases}
+$$
+
+### Syntax
+
+`result = ` [[stdlib_specialfunctions(module):leaky_relu(interface)]] ` (x,a)`
+
+### Class
+
+Elemental function
+
+### Arguments
+
+`x`: Shall be a scalar or array of any `real` kind. 
+`a`: Shall be a scalar or array of any `real` kind. 
+
+### Return value
+
+The function returns a value with the same type and kind as input argument.
+
+### Example
+```fortran
+{!example/specialfunctions_activations/example_leaky_relu.f90!}
+```
+
+## `Leaky_relu_grad` - Gradient of the Leaky Rectified Linear Unit function
+
+### Status
+
+Experimental
+
+### Description
+
+Computes the gradient of the leaky_relu function:
+$$
+\text{f}(x) =
+\begin{cases} 
+1, & \text{if } x \geq 0 \\
+a , & \text{otherwise}
+\end{cases}
+$$
+
+### Syntax
+
+`result = ` [[stdlib_specialfunctions(module):leaky_relu_grad(interface)]] ` (x,a)`
+
+### Class
+
+Elemental function
+
+### Arguments
+
+`x`: Shall be a scalar or array of any `real` kind.
+`a`: Shall be a scalar or array of any `real` kind.  
+
+### Return value
+
+The function returns a value with the same type and kind as the input argument.
+
 ## `Gelu` - Gaussian Error Linear Unit function
 
 ### Status

example/specialfunctions_activations/CMakeLists.txt

@@ -1,6 +1,7 @@
 ADD_EXAMPLE(elu)
 ADD_EXAMPLE(gaussian)
 ADD_EXAMPLE(gelu)
+ADD_EXAMPLE(leaky_relu)
 ADD_EXAMPLE(relu)
 ADD_EXAMPLE(selu)
 ADD_EXAMPLE(silu)

example/specialfunctions_activations/example_leaky_relu.f90

@@ -0,0 +1,13 @@
+program example_gelu
+    use stdlib_kinds, only: sp
+    use stdlib_math, only: linspace
+    use stdlib_specialfunctions, only: leaky_relu
+    implicit none
+    integer, parameter :: n = 10
+    real(sp) :: x(n), y(n)
+    
+    x = linspace(-2._sp, 2._sp, n)
+    y = leaky_relu( x , 0.1_sp )
+    print *, y
+end program example_gelu
+  

src/stdlib_specialfunctions.fypp

@@ -117,6 +117,36 @@ module stdlib_specialfunctions
     end interface
     public :: relu_grad
 
+    interface leaky_relu
+        !! Version: experimental
+        !!
+        !! Leaky Rectified linear unit function
+        !> ([Specification](../page/specs/stdlib_specialfunctions.html#leaky_relu))
+        #:for rk, rt in REAL_KINDS_TYPES
+        elemental module function leaky_relu_${rk}$( x , a ) result( y )
+            ${rt}$, intent(in) :: x
+            ${rt}$, intent(in) :: a
+            ${rt}$ :: y
+        end function
+        #:endfor
+    end interface
+    public :: leaky_relu
+
+    interface leaky_relu_grad
+        !! Version: experimental
+        !!
+        !! Gradient of the Leaky Rectified linear unit function
+        !> ([Specification](../page/specs/stdlib_specialfunctions.html#leaky_relu_grad))
+        #:for rk, rt in REAL_KINDS_TYPES
+        elemental module function leaky_relu_grad_${rk}$( x , a ) result( y )
+            ${rt}$, intent(in) :: x
+            ${rt}$, intent(in) :: a
+            ${rt}$ :: y
+        end function
+        #:endfor
+    end interface
+    public :: leaky_relu_grad
+
     interface gelu
         !! Version: experimental
         !!

src/stdlib_specialfunctions_activations.fypp

@@ -65,6 +65,26 @@ end function
 
 #:endfor
 
+!==================================================
+! Leaky Rectified Linear Unit
+!==================================================
+#:for rk, rt in REAL_KINDS_TYPES
+elemental module function Leaky_relu_${rk}$( x , a ) result( y )
+    ${rt}$, intent(in) :: x
+    ${rt}$, intent(in) :: a
+    ${rt}$ :: y
+    y = merge( x, a * x , x >= 0._${rk}$)
+end function
+
+elemental module function Leaky_relu_grad_${rk}$( x , a ) result( y )
+    ${rt}$, intent(in) :: x
+    ${rt}$, intent(in) :: a
+    ${rt}$ :: y
+    y = merge( 1._${rk}$ , a , x >= 0._${rk}$)
+end function
+
+#:endfor
+
 !==================================================
 ! GELU: Gaussian Error Linear Units function
 !==================================================

test/specialfunctions/test_specialfunctions_activations.fypp

@@ -24,6 +24,7 @@ contains
             new_unittest("gaussian", test_gaussian),        &
             new_unittest("elu", test_elu),                  &
             new_unittest("relu", test_relu),                &
+            new_unittest("leaky_relu", test_leaky_relu),    &
             new_unittest("gelu"   , test_gelu),             &
             new_unittest("selu"   , test_selu),             &
             new_unittest("sigmoid", test_sigmoid),          &
@@ -147,6 +148,24 @@ contains
         if (allocated(error)) return
     end subroutine
 
+    subroutine test_leaky_relu(error)
+        type(error_type), allocatable, intent(out) :: error
+        integer, parameter :: n = 10
+        real(sp) :: x(n), y(n), y_ref(n), a
+
+        call random_number(x)
+        a = 0.1_sp
+        where(x>=0._sp)
+            y_ref = x
+        elsewhere
+            y_ref = a * x
+        end where
+        y = Leaky_relu( x , a )
+
+        call check(error, norm2(y-y_ref) < n*tol_sp )
+        if (allocated(error)) return
+    end subroutine
+
     subroutine test_sigmoid(error)
         type(error_type), allocatable, intent(out) :: error
         integer, parameter :: n = 10