refactor augmentation to be a plugin

Author: bjarthur

configuration.py

@@ -73,6 +73,7 @@
 # neural network architecture to use
 architecture_plugin="convolutional"
 overlapped_prefix="not_"
+augmentation_plugin="volume-noise-dc-reverse-invert"
 
 # on what computer to do the computation
 default_where="local"

src/activations

@@ -157,10 +157,11 @@ def main():
   def infer_step(isound):
     # HACK: get_data not guaranteed to return isounds in order
     fingerprints, _, sounds = D.get_data(
-                                 FLAGS.batch_size, isound, model_settings,
-                                 FLAGS.loss, FLAGS.overlapped_prefix,
-                                 time_shift_tics, 'testing',
-                                 model.use_audio, model.use_video, video_findfile)
+            FLAGS.batch_size, isound, model_settings,
+            FLAGS.loss, FLAGS.overlapped_prefix,
+            time_shift_tics, 'testing',
+            model.use_audio, model.use_video, video_findfile,
+            None, None)
     hidden_activations, logits = thismodel(fingerprints, training=False)
     return fingerprints, sounds, logits, hidden_activations
 

src/data.py

@@ -411,14 +411,22 @@ def _get_data(q, o, how_many, offset, model_settings, loss, overlapped_prefix,
               shiftby, mode, use_audio, use_video, video_findfile,
               data_index, labels_list, np_rng, data_dir,
               audio_read_plugin, audio_read_plugin_kwargs,
-              video_read_plugin, video_read_plugin_kwargs):
+              video_read_plugin, video_read_plugin_kwargs,
+              augmentation_plugin, augmentation_parameters):
     q.cancel_join_thread()
 
     from lib import compute_background, load_audio_read_plugin, load_video_read_plugin
     load_audio_read_plugin(audio_read_plugin, audio_read_plugin_kwargs)
     load_video_read_plugin(video_read_plugin, video_read_plugin_kwargs)
     from lib import audio_read, video_read
 
+    if use_audio and mode=='training':
+        import importlib
+        sys.path.insert(0,os.path.dirname(augmentation_plugin))
+        tmp = importlib.import_module(os.path.basename(augmentation_plugin))
+        def augment(audio_slice, augmentation_parameters):
+            return tmp.augment(audio_slice, augmentation_parameters)
+
     while True:
         # Pick one of the partitions to choose sounds from.
         pick_deterministically = mode != 'training'
@@ -498,29 +506,8 @@ def _get_data(q, o, how_many, offset, model_settings, loss, overlapped_prefix,
                         labels[i - offset, labels_list.index(root)] = target
                         sounds[-1].append({k: v for k,v in overlapped_sound.items() if k!='overlaps'})
 
-        # augmentation
         if use_audio and mode=='training':
-            volume_range = [float(x) for x in model_settings['augment_volume'].split(',')]
-            noise_range = [float(x) for x in model_settings['augment_noise'].split(',')]
-            dc_range = [float(x) for x in model_settings['augment_dc'].split(',')]
-            reverse_bool = model_settings['augment_reverse'] == 'yes'
-            invert_bool = model_settings['augment_invert'] == 'yes'
-            if volume_range != [1,1]:
-                volume_ranges = np.random.uniform(*volume_range, (nsounds,1,audio_nchannels))
-                audio_slice *= volume_ranges
-            if noise_range != [0,0]:
-                noise_ranges = np.random.uniform(*noise_range, (nsounds,1,audio_nchannels))
-                noises = np.random.normal(0, noise_ranges, audio_slice.shape)
-                audio_slice += noises
-            if dc_range != [0,0]:
-                dc_ranges = np.random.uniform(*dc_range, (nsounds,1,audio_nchannels))
-                audio_slice += dc_ranges
-            if reverse_bool:
-                ireverse = np.random.choice([False,True], nsounds)
-                audio_slice[ireverse] = np.flip(audio_slice[ireverse], axis=1)
-            if invert_bool:
-                iinvert = np.random.choice([-1,1], (nsounds,1,1))
-                audio_slice *= iinvert
+            audio_slice = augment(audio_slice, augmentation_parameters)
 
         if loss=='autoencoder':
             labels = audio_slice
@@ -533,7 +520,8 @@ def _get_data(q, o, how_many, offset, model_settings, loss, overlapped_prefix,
             q.put([video_slice, labels, sounds])
 
 def get_data(how_many, offset, model_settings, loss, overlapped_prefix,
-             shiftby, mode, use_audio, use_video, video_findfile):
+             shiftby, mode, use_audio, use_video, video_findfile,
+             augmentation_plugin, augmentation_parameters):
     """Gather sounds from the data set, applying transformations as needed.
 
     When the mode is 'training', a random selection of sounds will be returned,
@@ -575,7 +563,8 @@ def get_data(how_many, offset, model_settings, loss, overlapped_prefix,
                           mode, use_audio, use_video, video_findfile,
                           data_index, labels_list, np_rng, data_dir,
                           audio_read_plugin, audio_read_plugin_kwargs,
-                          video_read_plugin, video_read_plugin_kwargs),
+                          video_read_plugin, video_read_plugin_kwargs,
+                          augmentation_plugin, augmentation_parameters),
                     daemon=True)
         p.start()
         processes[mode].append(p)

src/generalize

@@ -40,11 +40,8 @@
 #     --video_channels=0 \
 #     --batch_seed=_1 \
 #     --weights_seed=_1 \
-#     --augment_volume=1,1 \
-#     --augment_noise=0,0 \
-#     --augment_dc=0,0 \
-#     --augment_reverse=no \
-#     --augment_invert=no \
+#     --augmentation_plugin=volume-noise-dc-invert-reverse \
+#     --augmentation_parameters='{"volume":"1,1", "noise":"0,0", "dc:"0,0", "reverse":"no", "invert":"no"}' \
 #     --deterministic=0 \
 #     --igpu=0 \
 #     --ioffset=3 \
@@ -138,11 +135,8 @@ def main():
             "--video_channels="+FLAGS.video_channels,
             "--random_seed_batch="+str(FLAGS.batch_seed),
             "--random_seed_weights="+str(FLAGS.weights_seed),
-            "--augment_volume="+str(FLAGS.augment_volume),
-            "--augment_noise="+str(FLAGS.augment_noise),
-            "--augment_dc="+str(FLAGS.augment_dc),
-            "--augment_reverse="+str(FLAGS.augment_reverse),
-            "--augment_invert="+str(FLAGS.augment_invert),
+            "--augmentation_plugin="+FLAGS.augmentation_plugin,
+            "--augmentation_parameters="+FLAGS.augmentation_parameters.replace('<','^^^<').replace('>','^^^>'),
             "--deterministic="+FLAGS.deterministic,
             "--train_dir="+os.path.join(FLAGS.logdir,"generalize_"+model),
             "--summaries_dir="+os.path.join(FLAGS.logdir,"summaries_"+model),
@@ -305,30 +299,15 @@ if __name__ == '__main__':
       default=59185,
       help='Randomize weight initialization if -1; otherwise use supplied number as seed.')
   parser.add_argument(
-      '--augment_volume',
+      '--augmentation_plugin',
       type=str,
-      default='1,1',
-      help='Multiply each annotation by a uniform random number in this interval when training')
-  parser.add_argument(
-      '--augment_noise',
-      type=str,
-      default='0,0',
-      help='Add noise to each annotation with a uniform random std dev in this interval when training')
-  parser.add_argument(
-      '--augment_dc',
-      type=str,
-      default='0,0',
-      help='Add to each annotation a uniform random number in this interval when training')
-  parser.add_argument(
-      '--augment_reverse',
-      type=str,
-      default='no',
-      help='Flip in time with a probability of half each annotation when training')
+      default='{}',
+      help='What augmentation plugin to use')
   parser.add_argument(
-      '--augment_invert',
+      '--augmentation_parameters',
       type=str,
-      default='no',
-      help='Negate with a probability of half each annotation when training')
+      default='{}',
+      help='What augmentation parameters to use')
   parser.add_argument(
       '--model_architecture',
       type=str,

src/gui/controller.py

@@ -1224,11 +1224,8 @@ async def train_actuate():
                 "--video_channels="+str(M.video_channels), \
                 "--batch_seed="+V.batch_seed.value, \
                 "--weights_seed="+V.weights_seed.value, \
-                "--augment_volume="+V.augment_volume.value, \
-                "--augment_noise="+V.augment_noise.value, \
-                "--augment_dc="+V.augment_dc.value, \
-                "--augment_reverse="+V.augment_reverse.value, \
-                "--augment_invert="+V.augment_invert.value, \
+                "--augmentation_plugin="+M.augmentation_plugin, \
+                "--augmentation_parameters="+json.dumps({k:v.value for k,v in V.augmentation_parameters.items()}), \
                 "--deterministic="+M.deterministic, \
                 "--igpu=QUEUE1", \
                 "--ireplicates="+','.join([str(x) for x in range(ireplicate, min(1+nreplicates, \
@@ -1325,11 +1322,8 @@ async def leaveout_actuate(kind):
                 "--video_channels="+str(M.video_channels), \
                 "--batch_seed="+V.batch_seed.value, \
                 "--weights_seed="+V.weights_seed.value, \
-                "--augment_volume="+V.augment_volume.value, \
-                "--augment_noise="+V.augment_noise.value, \
-                "--augment_dc="+V.augment_dc.value, \
-                "--augment_reverse="+V.augment_reverse.value, \
-                "--augment_invert="+V.augment_invert.value, \
+                "--augmentation_plugin="+M.augmentation_plugin, \
+                "--augmentation_parameters="+json.dumps({k:v.value for k,v in V.augmentation_parameters.items()}), \
                 "--deterministic="+M.deterministic, \
                 "--ioffset="+str(ivalidation_file),
                 "--igpu=QUEUE1", \
@@ -1401,11 +1395,8 @@ async def xvalidate_actuate():
                 "--video_channels="+str(M.video_channels), \
                 "--batch_seed="+V.batch_seed.value, \
                 "--weights_seed="+V.weights_seed.value, \
-                "--augment_volume="+V.augment_volume.value, \
-                "--augment_noise="+V.augment_noise.value, \
-                "--augment_dc="+V.augment_dc.value, \
-                "--augment_reverse="+V.augment_reverse.value, \
-                "--augment_invert="+V.augment_invert.value, \
+                "--augmentation_plugin="+M.augmentation_plugin, \
+                "--augmentation_parameters="+json.dumps({k:v.value for k,v in V.augmentation_parameters.items()}), \
                 "--deterministic="+M.deterministic, \
                 "--igpu=QUEUE1", \
                 "--kfold="+V.kfold.value, \
@@ -2251,21 +2242,6 @@ def _copy_callback():
             elif "random_seed_weights = " in line:
                 m=re.search('random_seed_weights = (.*)', line)
                 V.weights_seed.value = m.group(1)
-            elif "augment_volume = " in line:
-                m=re.search('augment_volume = (.*)', line)
-                V.augment_volume.value = m.group(1)
-            elif "augment_noise = " in line:
-                m=re.search('augment_noise = (.*)', line)
-                V.augment_noise.value = m.group(1)
-            elif "augment_dc = " in line:
-                m=re.search('augment_dc = (.*)', line)
-                V.augment_dc.value = m.group(1)
-            elif "augment_reverse = " in line:
-                m=re.search('augment_reverse = (.*)', line)
-                V.augment_reverse.value = m.group(1)
-            elif "augment_invert = " in line:
-                m=re.search('augment_invert = (.*)', line)
-                V.augment_invert.value = m.group(1)
             elif "validate_step_period = " in line:
                 m=re.search('validate_step_period = (\d+)', line)
                 V.save_and_validate_period.value = m.group(1)
@@ -2312,6 +2288,11 @@ def _copy_callback():
                 params = json.loads(m.group(1).replace("'",'"'))
                 for k,v in params.items():
                   V.model_parameters[k].value = v
+            elif "augmentation_parameters = " in line:
+                m=re.search('augmentation_parameters = ({.*})', line)
+                params = json.loads(m.group(1).replace("'",'"'))
+                for k,v in params.items():
+                  V.augmentation_parameters[k].value = v
     _copy_callback_finalize()
 
 def copy_callback():

src/gui/main.py

@@ -38,6 +38,7 @@
 doubleclick_parameters = list(V.doubleclick_parameters.values())
 model_parameters = list(V.model_parameters.values())
 cluster_parameters = list(V.cluster_parameters.values())
+augmentation_parameters = list(V.augmentation_parameters.values())
 
 main_content = row(
         column(
@@ -128,12 +129,10 @@
                                for c in r])
                          for r in V.cluster_parameters_partitioned],
                        background="honeydew"),
-                column(row(V.augment_volume, V.augment_noise,
-                           width=M.gui_width_pix//11*2),
-                       row(V.augment_dc, V.augment_reverse,
-                           width=M.gui_width_pix//11*2),
-                       row(V.augment_invert,
-                           width=M.gui_width_pix//11),
+                column(*[row(*[column(augmentation_parameters[c],
+                                      width=round(M.gui_width_pix/11*V.augmentation_parameters_width[c]))
+                               for c in r])
+                         for r in V.augmentation_parameters_partitioned],
                        background="azure"),
                 column(*[row(*[column(model_parameters[c],
                                       width=round(M.gui_width_pix/11*V.model_parameters_width[c]))

src/gui/model.py

@@ -57,11 +57,6 @@ def save_state_callback():
                      'nreplicates': V.nreplicates.value,
                      'batch_seed': V.batch_seed.value,
                      'weights_seed': V.weights_seed.value,
-                     'augment_volume': V.augment_volume.value,
-                     'augment_noise': V.augment_noise.value,
-                     'augment_dc': V.augment_dc.value,
-                     'augment_reverse': V.augment_reverse.value,
-                     'augment_invert': V.augment_invert.value,
                      'labels': str.join(',',[x.value for x in V.label_texts]),
                      'file_dialog_string': V.file_dialog_string.value,
                      'context': V.context.value,
@@ -72,7 +67,8 @@ def save_state_callback():
                   **{k:v.value for k,v in V.detect_parameters.items()},
                   **{k:v.value for k,v in V.doubleclick_parameters.items()},
                   **{k:v.value for k,v in V.model_parameters.items()},
-                  **{k:v.value for k,v in V.cluster_parameters.items()}},
+                  **{k:v.value for k,v in V.cluster_parameters.items()},
+                  **{k:v.value for k,v in V.augmentation_parameters.items()}},
                  fid)
 
 def isannotated(sound):
@@ -253,7 +249,7 @@ def init(_bokeh_document, _configuration_file, _use_aitch):
     global user_changed_recording, user_copied_parameters
     global audio_read, audio_read_exts, audio_read_rec2ch, audio_read_strip_rec, trim_ext
     global video_read, detect_labels, doubleclick_annotation, context_data, context_data_istart, model, video_findfile
-    global detect_parameters, doubleclick_parameters, model_parameters, cluster_parameters
+    global detect_parameters, doubleclick_parameters, model_parameters, cluster_parameters, augmentation_parameters
 
     bokeh_document = _bokeh_document
 
@@ -293,6 +289,10 @@ def init(_bokeh_document, _configuration_file, _use_aitch):
     tmp = importlib.import_module(os.path.basename(cluster_plugin))
     cluster_parameters = tmp.cluster_parameters()
 
+    sys.path.insert(0,os.path.dirname(augmentation_plugin))
+    tmp = importlib.import_module(os.path.basename(augmentation_plugin))
+    augmentation_parameters = tmp.augmentation_parameters()
+
     sys.path.insert(0,os.path.dirname(video_findfile_plugin))
     video_findfile = importlib.import_module(os.path.basename(video_findfile_plugin)).video_findfile
 
@@ -499,11 +499,6 @@ def is_local_server_or_cluster(varname, varvalue):
                           'nreplicates':'1', \
                           'batch_seed':'-1', \
                           'weights_seed':'-1', \
-                          'augment_volume':'1,1', \
-                          'augment_noise':'0,0', \
-                          'augment_dc':'0,0', \
-                          'augment_reverse':'no', \
-                          'augment_invert':'no', \
                           'labels':','*(nlabels-1), \
                           'file_dialog_string':os.getcwd(), \
                           'context':str(0.2048 / time_scale), \
@@ -514,7 +509,8 @@ def is_local_server_or_cluster(varname, varvalue):
                        **{x[0]:x[3] for x in detect_parameters}, \
                        **{x[0]:x[3] for x in doubleclick_parameters}, \
                        **{x[0]:x[3] for x in model_parameters},
-                       **{x[0]:x[3] for x in cluster_parameters}},
+                       **{x[0]:x[3] for x in cluster_parameters},
+                       **{x[0]:x[3] for x in augmentation_parameters}},
                       fid)
 
     with open(statepath, 'r') as fid: