👊 Change adam->radam and use ExponentialDecay Learning rate.

Author: dathudeptrai

examples/parallel_wavegan/conf/parallel_wavegan.v1.yaml

@@ -76,18 +76,19 @@ is_shuffle: true               # shuffle dataset after each epoch.
 #             OPTIMIZER & SCHEDULER SETTING               #
 ###########################################################
 generator_optimizer_params:
-    lr_fn: "PiecewiseConstantDecay"
+    lr_fn: "ExponentialDecay"
     lr_params: 
-        boundaries: [100000]       # = discriminator_train_start_steps.
-        values: [0.0005, 0.0001]    # learning rate each interval.
+        initial_learning_rate: 0.0005
+        decay_steps: 200000
+        decay_rate: 0.5
 
 
 discriminator_optimizer_params:
-    lr_fn: "PiecewiseConstantDecay"
+    lr_fn: "ExponentialDecay"
     lr_params: 
-        boundaries: [0]             # after resume and start training discriminator, global steps is 100k, but local discriminator step is 0
-        values: [0.0001, 0.0001]    # learning rate each interval.
-
+        initial_learning_rate: 0.0005
+        decay_steps: 200000
+        decay_rate: 0.5
 
 ###########################################################
 #                    INTERVAL SETTING                     #

examples/parallel_wavegan/train_parallel_wavegan.py

@@ -50,6 +50,8 @@
 from tensorflow_tts.losses import TFMultiResolutionSTFT
 from tensorflow_tts.utils import calculate_2d_loss, calculate_3d_loss, return_strategy
 
+from tensorflow_addons.optimizers import RectifiedAdam
+
 
 class ParallelWaveganTrainer(GanBasedTrainer):
     """ParallelWaveGAN Trainer class based on GanBasedTrainer."""
@@ -439,10 +441,10 @@ def main():
             config["discriminator_optimizer_params"]["lr_fn"],
         )(**config["discriminator_optimizer_params"]["lr_params"])
 
-        gen_optimizer = tf.keras.optimizers.Adam(
+        gen_optimizer = RectifiedAdam(
             learning_rate=generator_lr_fn, amsgrad=False
         )
-        dis_optimizer = tf.keras.optimizers.Adam(
+        dis_optimizer = RectifiedAdam(
             learning_rate=discriminator_lr_fn, amsgrad=False
         )