disable dropout in inference mode (net % predict); TODO enable in net % train

Author: milancurcic

src/nf/nf_dropout_layer.f90

@@ -22,7 +22,7 @@ module nf_dropout_layer
 
     real :: dropout_rate ! probability of dropping a neuron
     real :: scale ! scale factor to preserve the input sum
-    logical :: training = .true.
+    logical :: training = .false. ! set to .true. in training mode
 
   contains
 
@@ -33,11 +33,13 @@ module nf_dropout_layer
   end type dropout_layer
 
   interface dropout_layer
-    module function dropout_layer_cons(rate) &
+    module function dropout_layer_cons(rate, training) &
       result(res)
       !! This function returns the `dropout_layer` instance.
       real, intent(in) :: rate
         !! Dropout rate
+      logical, intent(in), optional :: training
+        !! Training mode (default .false.)
       type(dropout_layer) :: res
         !! dropout_layer instance
     end function dropout_layer_cons

src/nf/nf_dropout_layer_submodule.f90

@@ -4,12 +4,12 @@
 
 contains
 
-  module function dropout_layer_cons(rate) result(res)
+  module function dropout_layer_cons(rate, training) result(res)
     real, intent(in) :: rate
+    logical, intent(in), optional :: training
     type(dropout_layer) :: res
-
-    ! Initialize dropout rate
     res % dropout_rate = rate
+    if (present(training)) res % training = training
   end function dropout_layer_cons
 
 
@@ -36,19 +36,27 @@ module subroutine forward(self, input)
     class(dropout_layer), intent(in out) :: self
     real, intent(in) :: input(:)
 
-    ! Generate random mask for dropout
-    call random_number(self % mask)
-    where (self % mask < self % dropout_rate)
-      self % mask = 0
-    elsewhere
-      self % mask = 1
-    end where
+    ! Generate random mask for dropout, training mode only
+    if (self % training) then
+
+      call random_number(self % mask)
+      where (self % mask < self % dropout_rate)
+        self % mask = 0
+      elsewhere
+        self % mask = 1
+      end where
+
+      ! Scale factor to preserve the input sum
+      self % scale = sum(input) / sum(input * self % mask)
+
+      ! Apply dropout mask
+      self % output = input * self % mask * self % scale
 
-    ! Scale factor to preserve the input sum
-    self % scale = sum(input) / sum(input * self % mask)
+    else
+      ! In inference mode, we don't apply dropout; simply pass through the input
+      self % output = input
 
-    ! Apply dropout mask
-    self % output = input * self % mask * self % scale
+    end if
 
   end subroutine forward
 

src/nf/nf_layer_constructors.f90

@@ -85,7 +85,7 @@ module function dense(layer_size, activation) result(res)
         !! Resulting layer instance
     end function dense
 
-    module function dropout(rate) result(res)
+    module function dropout(rate, training) result(res)
       !! Create a dropout layer with a given dropout rate.
       !!
       !! This layer is for randomly disabling neurons during training.
@@ -99,6 +99,8 @@ module function dropout(rate) result(res)
       !! ```
       real, intent(in) :: rate
         !! Dropout rate - fraction of neurons to randomly disable during training
+      logical, intent(in), optional :: training
+        !! Training mode (default .false.)
       type(layer) :: res
         !! Resulting layer instance
     end function dropout

src/nf/nf_layer_constructors_submodule.f90

@@ -64,11 +64,12 @@ module function dense(layer_size, activation) result(res)
   end function dense
 
 
-  module function dropout(rate) result(res)
+  module function dropout(rate, training) result(res)
     real, intent(in) :: rate
+    logical, intent(in), optional :: training
     type(layer) :: res
     res % name = 'dropout'
-    allocate(res % p, source=dropout_layer(rate))
+    allocate(res % p, source=dropout_layer(rate, training))
   end function dropout
 
 

src/nf/nf_layer_submodule.f90

@@ -304,7 +304,7 @@ elemental module function get_num_params(self) result(num_params)
       type is (dense_layer)
         num_params = this_layer % get_num_params()
       type is (dropout_layer)
-        num_params = size(this_layer % mask)
+        num_params = 0
       type is (conv2d_layer)
         num_params = this_layer % get_num_params()
       type is (maxpool2d_layer)

src/nf/nf_network_submodule.f90

@@ -221,10 +221,19 @@ module function predict_1d(self, input) result(res)
     class(network), intent(in out) :: self
     real, intent(in) :: input(:)
     real, allocatable :: res(:)
-    integer :: num_layers
+    integer :: n, num_layers
 
     num_layers = size(self % layers)
 
+    ! predict is run in inference mode only;
+    ! set all dropout layers' training mode to false.
+    do n = 2, num_layers
+      select type(this_layer => self % layers(n) % p)
+        type is(dropout_layer)
+          this_layer % training = .false.
+      end select
+    end do
+
     call self % forward(input)
 
     select type(output_layer => self % layers(num_layers) % p)
@@ -245,10 +254,19 @@ module function predict_3d(self, input) result(res)
     class(network), intent(in out) :: self
     real, intent(in) :: input(:,:,:)
     real, allocatable :: res(:)
-    integer :: num_layers
+    integer :: n, num_layers
 
     num_layers = size(self % layers)
 
+    ! predict is run in inference mode only;
+    ! set all dropout layers' training mode to false.
+    do n = 2, num_layers
+      select type(this_layer => self % layers(n) % p)
+        type is(dropout_layer)
+          this_layer % training = .false.
+      end select
+    end do
+
     call self % forward(input)
 
     select type(output_layer => self % layers(num_layers) % p)