simplify dropout in convolutional architecture

Author: bjarthur

src/convolutional.py

@@ -189,11 +189,7 @@ def model_parameters(time_units, freq_units, time_scale, freq_scale):
         ["nconvlayers",     "# conv layers",            '',                   '2',                    1, [],                 nlayers_callback,                                              True],
         ["kernel_sizes",    "kernels",                  '',                   '5x5,3',                1, [],                 None,                                                          True],
         ["nfeatures",       "# features",               '',                   '64,64',                1, [],                 None,                                                          True],
-        ["dropout_kind",    "dropout kind",             ['none',
-                                                         'unit'],             'unit',                 1, [],                 None,                                                          True],
-        ["dropout_rate",    "dropout %",                '',                   '50',                   1, ["dropout_kind",
-                                                                                                          ["unit",
-                                                                                                           "map"]],          None,                                                          True],
+        ["dropout",         "dropout %",                '',                   '50',                   1, [],                 None,                                                          True],
         ["normalization",   "normalization",            ['none',
                                                          'batch before ReLU',
                                                          'batch after ReLU'], 'none',                 1, [],                 None,                                                          True],
@@ -388,12 +384,7 @@ def create_model(model_settings, model_parameters, io=sys.stdout):
   dilate_time = parse_layers(model_parameters['dilate_time'], nconvlayers)
   dilate_freq = parse_layers(model_parameters['dilate_freq'], nconvlayers)
   use_residual = model_parameters['connection_type']=='residual'
-  dropout_rate = float(model_parameters['dropout_rate'])/100
-  if model_parameters['dropout_kind']=='unit':
-    dropout_kind = Dropout
-  else:
-    def Identity(x): return lambda x: x
-    dropout_kind = Identity
+  dropout = float(model_parameters['dropout'])/100
   if model_parameters['pool_kind']=='max':
     pool_kind = MaxPool2D
   elif model_parameters['pool_kind']=='average':
@@ -543,7 +534,8 @@ def Identity(x): return lambda x: x
   print("receptive_field_freq = %d bins = %f %s" % (receptive_field[1],
       receptive_field[1] * audio_tic_rate / window_tics / freq_scale, freq_units), file=io)
 
-  x = dropout_kind(dropout_rate)(x)
+  if dropout>0:
+      x = Dropout(dropout)(x)
 
   if pool_kind:
     x = pool_kind(pool_size=pool_size, strides=pool_size)(x)
@@ -553,8 +545,9 @@ def Identity(x): return lambda x: x
   # final dense layers (or actually, pan-freq pan-time 2D convs)
   for idense, nunits in enumerate(denselayers+[model_settings['nlabels']]):
     if idense>0:
-      relu = ReLU()(x)
-      x = dropout_kind(dropout_rate)(relu)
+      x = ReLU()(x)
+      if dropout>0:
+          x = Dropout(dropout)(x)
     x = Conv2D(nunits, (noutput_tics if idense==0 else 1, x_shape[2]))(x)
     hidden_layers.append(conv)
     x_shape = x.get_shape().as_list()

src/ensemble-concat-dense.py

@@ -24,7 +24,7 @@
                                                'both',
                                                'neither'], 'neither',  1, [], None, False],
     ["dense_layers", "dense",                 '',          '',         1, [], None, False],
-    ["dropout_rate", "dropout %",             '',          '0',        1, [], None, True],
+    ["dropout",      "dropout %",             '',          '0',        1, [], None, True],
     ]
 
 def load_model(model_settings, ckpt):
@@ -69,7 +69,7 @@ def create_model(model_settings, model_parameters):
     trainable = model_parameters['trainable']
     dense_layers = [] if model_parameters['dense_layers']=='' \
                    else [int(x) for x in model_parameters['dense_layers'].split(',')]
-    dropout_rate = float(model_parameters['dropout_rate'])/100
+    dropout = float(model_parameters['dropout'])/100
 
     hidden_layers = []
 
@@ -88,8 +88,8 @@ def create_model(model_settings, model_parameters):
     for idense, nunits in enumerate(dense_layers+[model_settings['nlabels']]):
         if idense>0:
             x = ReLU()(x)
-        if dropout_rate>0:
-            x = Dropout(dropout_rate)(x)
+            if dropout>0:
+                x = Dropout(dropout)(x)
         x = Conv1D(nunits, 1)(x)
         hidden_layers.append(x)
 

src/ensemble-transfer.py

@@ -22,12 +22,12 @@
 
 model_parameters = [
     # key, title in GUI, '' for textbox or [] for pull-down, default value, width, enable logic, callback, required
-    ["ckpt_files",    "checkpoint file(s)", '', '',  6, [], None, True],
-    ["trainable",     "trainable?",         '', '',  1, [], None, False],
-    ["splice_layers", "layer(s)",           '', '',  1, [], None, False],
-    ["conv_layers",   "conv",               '', '',  2, [], None, False],
-    ["dense_layers",  "dense",              '', '',  1, [], None, False],
-    ["dropout_rate",  "dropout %",          '', '0', 1, [], None, True],
+    ["ckpt_files",    "checkpoint file(s)", '', '',   6, [], None, True],
+    ["trainable",     "trainable?",         '', '',   1, [], None, False],
+    ["splice_layers", "layer(s)",           '', '',   1, [], None, False],
+    ["conv_layers",   "conv",               '', '',   2, [], None, False],
+    ["dense_layers",  "dense",              '', '',   1, [], None, False],
+    ["dropout",       "dropout %",          '', '50', 1, [], None, True],
     ]
 
 def load_model(context0, audio_tic_rate0, parallelize0, ckpt_file, io):
@@ -155,7 +155,7 @@ def create_model(model_settings, model_parameters, io=sys.stdout):
                         for x in model_parameters['conv_layers'].split(',')]
     dense_layers = [] if model_parameters['dense_layers']=='' \
                    else [int(x) for x in model_parameters['dense_layers'].split(',')]
-    dropout_rate = float(model_parameters['dropout_rate'])/100
+    dropout = float(model_parameters['dropout'])/100
 
     hidden_layers = []
 
@@ -190,8 +190,8 @@ def create_model(model_settings, model_parameters, io=sys.stdout):
         if fs==model_settings['audio_tic_rate']:
             x = Identity()
         else:
-            l = Resample(model_settings['audio_tic_rate'], fs)
-        x = l(inputs)
+            x = Resample(model_settings['audio_tic_rate'], fs)
+        x = x(inputs)
         x = m(x)
         hidden_layers.extend(x[0])
         lowerlegs.append(tf.keras.Model(inputs=inputs, outputs=x[1]))
@@ -210,8 +210,8 @@ def create_model(model_settings, model_parameters, io=sys.stdout):
     for idense, nunits in enumerate(dense_layers+[model_settings['nlabels']]):
         if idense>0:
             x = ReLU()(x)
-        if dropout_rate>0:
-            x = Dropout(dropout_rate)(x)
+            if dropout>0:
+                x = Dropout(dropout)(x)
         x = Conv1D(nunits, 1)(x)
         hidden_layers.append(x)
 

test/freeze-classify.py

@@ -46,8 +46,7 @@
 V.shiftby.value = "0.0"
 V.optimizer.value = "Adam"
 V.learning_rate.value = "0.0002"
-V.model_parameters["dropout_kind"].value = "unit"
-V.model_parameters["dropout_rate"].value = "50"
+V.model_parameters["dropout"].value = "50"
 V.model_parameters["augment_volume"].value = "1,1"
 V.model_parameters["augment_noise"].value = "0,0"
 V.model_parameters["normalization"].value = "none"

test/seeds.py

@@ -43,8 +43,7 @@
 V.shiftby.value = "0.0"
 V.optimizer.value = "Adam"
 V.learning_rate.value = "0.0002"
-V.model_parameters["dropout_kind"].value = "unit"
-V.model_parameters["dropout_rate"].value = "50"
+V.model_parameters["dropout"].value = "50"
 V.model_parameters["augment_volume"].value = "1,1"
 V.model_parameters["augment_noise"].value = "0,0"
 V.model_parameters["normalization"].value = "none"

test/shiftby.py

@@ -47,8 +47,7 @@
                          "nconvlayers": "2", \
                          "kernel_sizes": "3x3,32", \
                          "nfeatures": "8,8", \
-                         "dropout_kind": "unit", \
-                         "dropout_rate": "50", \
+                         "dropout": "50", \
                          "normalization": "none", \
                          "stride_time": "2", \
                          "stride_freq": "", \

test/tutorial.py

@@ -65,8 +65,7 @@
 V.shiftby.value = "0.0"
 V.optimizer.value = "Adam"
 V.learning_rate.value = "0.0002"
-V.model_parameters["dropout_kind"].value = "unit"
-V.model_parameters["dropout_rate"].value = "50"
+V.model_parameters["dropout"].value = "50"
 V.model_parameters["augment_volume"].value = "1,1"
 V.model_parameters["augment_noise"].value = "0,0"
 V.model_parameters["normalization"].value = "none"

test/tutorial.sh

@@ -74,7 +74,7 @@ shiftby=0.0
 optimizer=Adam
 learning_rate=0.0002
 architecture=convolutional
-model_parameters='{"representation":"mel-cepstrum","window":"6.4","stride":"1.6","mel_dct":"7,7","range":"","dropout_kind":"unit","dropout_rate":"50","augment_volume":"1,1","augment_noise":"0,0","normalization":"none","kernel_sizes":"5x5,3","nconvlayers":"2","denselayers":"","nfeatures":"64,64","stride_time":"","stride_freq":"","dilate_time":"","dilate_freq":"","pool_kind":"none","pool_size":"","connection_type":"plain"}'
+model_parameters='{"representation":"mel-cepstrum","window":"6.4","stride":"1.6","mel_dct":"7,7","range":"","dropout":"50","augment_volume":"1,1","augment_noise":"0,0","normalization":"none","kernel_sizes":"5x5,3","nconvlayers":"2","denselayers":"","nfeatures":"64,64","stride_time":"","stride_freq":"","dilate_time":"","dilate_freq":"","pool_kind":"none","pool_size":"","connection_type":"plain"}'
 logdir=$repo_path/test/scratch/tutorial-sh/trained-classifier1
 data_dir=$repo_path/test/scratch/tutorial-sh/groundtruth-data
 labels_touse=mel-pulse,mel-sine,ambient

test/visualization.py

@@ -58,8 +58,7 @@
 V.shiftby.value = "0.0"
 V.optimizer.value = "Adam"
 V.learning_rate.value = "0.0002"
-V.model_parameters["dropout_kind"].value = "unit"
-V.model_parameters["dropout_rate"].value = "50"
+V.model_parameters["dropout"].value = "50"
 V.model_parameters["augment_volume"].value = "1,1"
 V.model_parameters["augment_noise"].value = "0,0"
 V.model_parameters["normalization"].value = "none"