Linear2d layer

CMakeLists.txt

@@ -38,6 +38,8 @@ add_library(neural-fortran
   src/nf/nf_layer_constructors_submodule.f90
   src/nf/nf_layer.f90
   src/nf/nf_layer_submodule.f90
+  src/nf/nf_linear2d_layer.f90
+  src/nf/nf_linear2d_layer_submodule.f90
   src/nf/nf_loss.f90
   src/nf/nf_loss_submodule.f90
   src/nf/nf_maxpool2d_layer.f90

README.md

@@ -34,6 +34,7 @@ Read the paper [here](https://arxiv.org/abs/1902.06714).
 | Convolutional (2-d) | `conv2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅(*) |
 | Max-pooling (2-d) | `maxpool2d` | `input3d`, `conv2d`, `maxpool2d`, `reshape` | 3 | ✅ | ✅ |
 | Flatten | `flatten` | `input2d`, `input3d`, `conv2d`, `maxpool2d`, `reshape` | 1 | ✅ | ✅ |
+| Linear (2-d) | `linear2d` | `input2d` | 2 | ✅ | ✅ |
 | Reshape (1-d to 3-d) | `reshape` | `input1d`, `dense`, `flatten` | 3 | ✅ | ✅ |
 
 (*) See Issue [#145](https://github.com/modern-fortran/neural-fortran/issues/145) regarding non-converging CNN training on the MNIST dataset.

example/CMakeLists.txt

@@ -2,6 +2,7 @@ foreach(execid
   cnn_mnist
   dense_mnist
   get_set_network_params
+  linear2d
   network_parameters
   simple
   sine

example/linear2d.f90

@@ -0,0 +1,29 @@
+program linear2d_example
+
+  use nf, only: input, network, sgd, linear2d, mse, flatten
+  implicit none
+
+  type(network) :: net
+  real :: x(3, 4) = reshape( &
+    [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2], &
+    [3, 4])
+  real :: y(3) = [0.12, 0.1, 0.3]
+  integer, parameter :: num_iterations = 500
+  integer :: n
+
+  net = network([ &
+    input(3, 4), &
+    linear2d(3, 4, 1), &
+    flatten() &
+  ])
+
+  call net % print_info()
+
+  do n = 1, num_iterations
+    call net % forward(x)
+    call net % backward(y, mse())
+    call net % update(optimizer=sgd(learning_rate=1.))
+    print '(i4,3(3x,f8.6))', n, net % predict(x)
+  end do
+
+end program linear2d_example

src/nf.f90

@@ -3,7 +3,7 @@ module nf
   use nf_datasets_mnist, only: label_digits, load_mnist
   use nf_layer, only: layer
   use nf_layer_constructors, only: &
-    conv2d, dense, flatten, input, maxpool2d, reshape
+    conv2d, dense, flatten, input, maxpool2d, reshape, linear2d
   use nf_loss, only: mse, quadratic
   use nf_metrics, only: corr, maxabs
   use nf_network, only: network

src/nf/nf_layer_constructors.f90

@@ -8,7 +8,7 @@ module nf_layer_constructors
   implicit none
 
   private
-  public :: conv2d, dense, flatten, input, maxpool2d, reshape
+  public :: conv2d, dense, flatten, input, maxpool2d, reshape, linear2d
 
   interface input
 
@@ -185,6 +185,11 @@ module function reshape(output_shape) result(res)
         !! Resulting layer instance
     end function reshape
 
+    module function linear2d(sequence_length, in_features, out_features) result(res)
+      integer, intent(in) :: sequence_length, in_features, out_features
+      type(layer) :: res
+    end function linear2d
+
   end interface
 
 end module nf_layer_constructors

src/nf/nf_layer_constructors_submodule.f90

@@ -9,6 +9,7 @@
   use nf_input3d_layer, only: input3d_layer
   use nf_maxpool2d_layer, only: maxpool2d_layer
   use nf_reshape_layer, only: reshape3d_layer
+  use nf_linear2d_layer, only: linear2d_layer
   use nf_activation, only: activation_function, relu, sigmoid
 
   implicit none
@@ -71,6 +72,7 @@ module function flatten() result(res)
   end function flatten
 
 
+
   module function input1d(layer_size) result(res)
     integer, intent(in) :: layer_size
     type(layer) :: res
@@ -148,4 +150,13 @@ module function reshape(output_shape) result(res)
 
   end function reshape
 
+  module function linear2d(sequence_length, in_features, out_features) result(res)
+    integer, intent(in) :: sequence_length, in_features, out_features
+    type(layer) :: res
+
+    res % name = 'linear2d'
+    res % layer_shape = [sequence_length, out_features]
+    allocate(res % p, source=linear2d_layer(sequence_length, in_features, out_features))
+  end function linear2d
+
 end submodule nf_layer_constructors_submodule

src/nf/nf_layer_submodule.f90

@@ -9,6 +9,7 @@
   use nf_input3d_layer, only: input3d_layer
   use nf_maxpool2d_layer, only: maxpool2d_layer
   use nf_reshape_layer, only: reshape3d_layer
+  use nf_linear2d_layer, only: linear2d_layer
   use nf_optimizers, only: optimizer_base_type
 
 contains
@@ -47,6 +48,8 @@ pure module subroutine backward_1d(self, previous, gradient)
             call this_layer % backward(prev_layer % output, gradient)
           type is(maxpool2d_layer)
             call this_layer % backward(prev_layer % output, gradient)
+          type is(linear2d_layer)
+            call this_layer % backward(prev_layer % output, gradient)
         end select
 
     end select
@@ -60,9 +63,19 @@ pure module subroutine backward_2d(self, previous, gradient)
     class(layer), intent(in) :: previous
     real, intent(in) :: gradient(:,:)
 
-    ! Backward pass from a 2-d layer downstream currently implemented
-    ! only for dense and flatten layers
-    ! CURRENTLY NO LAYERS, tbd: pull/197 and pull/199
+    select type(this_layer => self % p)
+
+      type is(linear2d_layer)
+
+        select type(prev_layer => previous % p)
+          type is(input2d_layer)
+            call this_layer % backward(prev_layer % output, gradient)
+          type is(linear2d_layer)
+            call this_layer % backward(prev_layer % output, gradient)
+        end select
+
+    end select
+
   end subroutine backward_2d
 
 
@@ -182,6 +195,8 @@ pure module subroutine forward(self, input)
             call this_layer % forward(prev_layer % output)
           type is(reshape3d_layer)
             call this_layer % forward(prev_layer % output)
+          type is(linear2d_layer)
+            call this_layer % forward(prev_layer % output)
         end select
 
       type is(reshape3d_layer)
@@ -196,6 +211,16 @@ pure module subroutine forward(self, input)
             call this_layer % forward(prev_layer % output)
         end select
 
+      type is(linear2d_layer)
+
+        ! Upstream layers permitted: input2d, linear2d
+        select type(prev_layer => input % p)
+          type is(input2d_layer)
+            call this_layer % forward(prev_layer % output)
+          type is(linear2d_layer)
+            call this_layer % forward(prev_layer % output)
+        end select
+
     end select
 
   end subroutine forward
@@ -231,8 +256,10 @@ pure module subroutine get_output_2d(self, output)
 
       type is(input2d_layer)
         allocate(output, source=this_layer % output)
+      type is(linear2d_layer)
+        allocate(output, source=this_layer % output)
       class default
-        error stop '1-d output can only be read from an input1d, dense, or flatten layer.'
+        error stop '2-d output can only be read from an input2d or linear2d layer.'
 
     end select
 
@@ -285,7 +312,7 @@ impure elemental module subroutine init(self, input)
         self % layer_shape = shape(this_layer % output)
     end select
 
-    self % input_layer_shape = input % layer_shape 
+    self % input_layer_shape = input % layer_shape
     self % initialized = .true.
 
   end subroutine init
@@ -328,6 +355,8 @@ elemental module function get_num_params(self) result(num_params)
         num_params = 0
       type is (reshape3d_layer)
         num_params = 0
+      type is (linear2d_layer)
+        num_params = this_layer % get_num_params()
       class default
         error stop 'Unknown layer type.'
     end select
@@ -355,6 +384,8 @@ module function get_params(self) result(params)
         ! No parameters to get.
       type is (reshape3d_layer)
         ! No parameters to get.
+      type is (linear2d_layer)
+        params = this_layer % get_params()
       class default
         error stop 'Unknown layer type.'
     end select
@@ -379,9 +410,11 @@ module function get_gradients(self) result(gradients)
       type is (maxpool2d_layer)
         ! No gradients to get.
       type is (flatten_layer)
-        ! No gradients to get.
+        ! No parameters to get.
       type is (reshape3d_layer)
         ! No gradients to get.
+      type is (linear2d_layer)
+        gradients = this_layer % get_gradients()
       class default
         error stop 'Unknown layer type.'
     end select
@@ -429,6 +462,9 @@ module subroutine set_params(self, params)
       type is (conv2d_layer)
         call this_layer % set_params(params)
 
+      type is (linear2d_layer)
+        call this_layer % set_params(params)
+
       type is (maxpool2d_layer)
         ! No parameters to set.
         write(stderr, '(a)') 'Warning: calling set_params() ' &
@@ -446,6 +482,7 @@ module subroutine set_params(self, params)
 
           class default
         error stop 'Unknown layer type.'
+
     end select
 
   end subroutine set_params

src/nf/nf_linear2d_layer.f90

@@ -0,0 +1,79 @@
+module nf_linear2d_layer
+
+  use nf_activation, only: activation_function
+  use nf_base_layer, only: base_layer
+
+  implicit none
+
+  private
+  public :: linear2d_layer
+
+  type, extends(base_layer) :: linear2d_layer
+    integer :: sequence_length, in_features, out_features, batch_size
+
+    real, allocatable :: weights(:, :)
+    real, allocatable :: biases(:)
+    real, allocatable :: output(:, :)
+    real, allocatable :: gradient(:, :) ! input gradient
+    real, allocatable :: dw(:, :) ! weight gradients
+    real, allocatable :: db(:) ! bias gradients
+
+  contains
+
+    procedure :: backward
+    procedure :: forward
+    procedure :: init
+    procedure :: get_num_params
+    procedure :: get_params
+    procedure :: get_gradients
+    procedure :: set_params
+
+  end type linear2d_layer
+
+  interface linear2d_layer
+    module function linear2d_layer_cons(&
+        sequence_length, in_features, out_features&
+    ) result(res)
+      integer, intent(in) :: sequence_length, in_features, out_features
+      type(linear2d_layer) :: res
+    end function linear2d_layer_cons
+  end interface linear2d_layer
+
+  interface
+    pure module subroutine forward(self, input)
+      class(linear2d_layer), intent(in out) :: self
+      real, intent(in) :: input(:, :)
+    end subroutine forward
+
+    pure module subroutine backward(self, input, gradient)
+      class(linear2d_layer), intent(in out) :: self
+      real, intent(in) :: input(:, :)
+      real, intent(in) :: gradient(:, :)
+    end subroutine backward
+
+    module subroutine init(self, input_shape)
+      class(linear2d_layer), intent(in out) :: self
+      integer, intent(in) :: input_shape(:)
+    end subroutine init
+
+    pure module function get_num_params(self) result(num_params)
+       class(linear2d_layer), intent(in) :: self
+       integer :: num_params
+    end function get_num_params
+
+    module function get_params(self) result(params)
+      class(linear2d_layer), intent(in), target :: self
+      real, allocatable :: params(:)
+    end function get_params
+
+    module function get_gradients(self) result(gradients)
+      class(linear2d_layer), intent(in), target :: self
+      real, allocatable :: gradients(:)
+    end function get_gradients
+
+    module subroutine set_params(self, params)
+      class(linear2d_layer), intent(in out) :: self
+      real, intent(in), target :: params(:)
+    end subroutine set_params
+  end interface
+end module nf_linear2d_layer