update chinese example

Author: azraelkuan

examples/multiband_melgan/decode_mb_melgan.py

@@ -110,14 +110,14 @@ def main():
 
     # define model and load checkpoint
     mb_melgan = TFMelGANGenerator(
-        config=MultiBandMelGANGeneratorConfig(**config["multiband_melgan_generator"]),
+        config=MultiBandMelGANGeneratorConfig(**config["multiband_melgan_generator_params"]),
         name="multiband_melgan_generator",
     )
     mb_melgan._build()
     mb_melgan.load_weights(args.checkpoint)
 
     pqmf = TFPQMF(
-        config=MultiBandMelGANGeneratorConfig(**config["multiband_melgan_generator"]), name="pqmf"
+        config=MultiBandMelGANGeneratorConfig(**config["multiband_melgan_generator_params"]), name="pqmf"
     )
 
     for data in tqdm(dataset, desc="[Decoding]"):

examples/tacotron2/conf/tacotron2.v1.baker.yaml

@@ -32,7 +32,7 @@ tacotron2_params:
     prenet_activation: 'relu'
     prenet_dropout_rate: 0.5
     n_lstm_decoder: 1
-    reduction_factor: 1
+    reduction_factor: 2
     decoder_lstm_units: 1024
     attention_dim: 128
     attention_filters: 32

examples/tacotron2/decode_tacotron2.py

@@ -110,11 +110,13 @@ def main():
 
     # define data-loader
     dataset = CharactorMelDataset(
+        dataset=config["tacotron2_params"]["dataset"],
         root_dir=args.rootdir,
         charactor_query=char_query,
         mel_query=mel_query,
         charactor_load_fn=char_load_fn,
         mel_load_fn=mel_load_fn,
+        reduction_factor=config["tacotron2_params"]["reduction_factor"]
     )
     dataset = dataset.create(allow_cache=True, batch_size=args.batch_size)
 

examples/tacotron2/extract_duration.py

@@ -124,11 +124,13 @@ def main():
 
     # define data-loader
     dataset = CharactorMelDataset(
+        dataset=config["tacotron2_params"]["dataset"],
         root_dir=args.rootdir,
         charactor_query=char_query,
         mel_query=mel_query,
         charactor_load_fn=char_load_fn,
         mel_load_fn=mel_load_fn,
+        reduction_factor=config["tacotron2_params"]["reduction_factor"]
     )
     dataset = dataset.create(allow_cache=True, batch_size=args.batch_size)
 
@@ -146,6 +148,8 @@ def main():
         input_lengths = data["input_lengths"]
         mel_lengths = data["mel_lengths"]
         utt_ids = utt_ids.numpy()
+        real_mel_lengths = data["real_mel_lengths"]
+        del data["real_mel_lengths"]
 
         # tacotron2 inference.
         mel_outputs, post_mel_outputs, stop_outputs, alignment_historys = tacotron2(
@@ -163,10 +167,26 @@ def main():
             real_char_length = (
                 input_lengths[i].numpy() - 1
             )  # minus 1 because char have eos tokens.
-            real_mel_length = mel_lengths[i].numpy()
-            alignment = alignment[:real_char_length, :real_mel_length]
+            real_mel_length = real_mel_lengths[i].numpy()
+            alignment_mel_length = int(np.ceil(real_mel_length / config["tacotron2_params"]["reduction_factor"]))
+            alignment = alignment[:real_char_length, :alignment_mel_length]
             d = get_duration_from_alignment(alignment)  # [max_char_len]
 
+            d = d * config["tacotron2_params"]["reduction_factor"]
+            assert np.sum(d) >= real_mel_length, f"{d}, {np.sum(d)}, {alignment_mel_length}, {real_mel_length}"
+            if np.sum(d) > real_mel_length:
+                rest = np.sum(d) - real_mel_length
+                # print(d, np.sum(d), real_mel_length)
+                if d[-1] > rest:
+                    d[-1] -= rest
+                elif d[0] > rest:
+                    d[0] -= rest
+                else:
+                    d[-1] -= rest // 2
+                    d[0] -= (rest - rest // 2)
+
+                assert d[-1] > 0 and d[0] > 0, f'{d}, {np.sum(d)}, {real_mel_length}'
+
             saved_name = utt_ids[i].decode("utf-8")
 
             # check a length compatible

examples/tacotron2/tacotron_dataset.py

@@ -23,7 +23,9 @@
 import tensorflow as tf
 
 from tensorflow_tts.datasets.abstract_dataset import AbstractDataset
-from tensorflow_tts.processor.ljspeech import symbols
+from tensorflow_tts.processor.ljspeech import symbols as ljspeech_symbols
+from tensorflow_tts.utils.korean import symbols as kss_symbols
+from tensorflow_tts.processor.baker import symbols as baker_symbols
 from tensorflow_tts.utils import find_files
 
 
@@ -51,6 +53,7 @@ class CharactorMelDataset(AbstractDataset):
 
     def __init__(
         self,
+        dataset,
         root_dir,
         charactor_query="*-ids.npy",
         mel_query="*-norm-feats.npy",
@@ -100,6 +103,13 @@ def __init__(
             suffix = charactor_query[1:]
             utt_ids = [os.path.basename(f).replace(suffix, "") for f in charactor_files]
 
+        eos_token_dict = {
+            "ljspeech": len(ljspeech_symbols) - 1,
+            "kss": len(kss_symbols) - 1,
+            "baker": len(baker_symbols) - 1
+        }
+        self.eos_token_id = eos_token_dict[dataset]
+
         # set global params
         self.utt_ids = utt_ids
         self.mel_files = mel_files
@@ -139,10 +149,11 @@ def generator(self, utt_ids):
             char_length = self.char_lengths[i]
 
             # add eos token for charactor since charactor is original token.
-            charactor = np.concatenate([charactor, [len(symbols) - 1]], -1)
+            charactor = np.concatenate([charactor, [self.eos_token_id]], -1)
             char_length += 1
 
             # padding mel to make its length is multiple of reduction factor.
+            real_mel_length = mel_length
             remainder = mel_length % self.reduction_factor
             if remainder != 0:
                 new_mel_length = mel_length + self.reduction_factor - remainder
@@ -169,6 +180,7 @@ def generator(self, utt_ids):
                 "speaker_ids": 0,
                 "mel_gts": mel,
                 "mel_lengths": mel_length,
+                "real_mel_lengths": real_mel_length,
                 "g_attentions": g_attention,
             }
 
@@ -209,6 +221,7 @@ def create(
             "speaker_ids": 0,
             "mel_gts": self.mel_pad_value,
             "mel_lengths": 0,
+            "real_mel_lengths": 0,
             "g_attentions": self.ga_pad_value,
         }
 
@@ -224,6 +237,7 @@ def create(
             if self.use_fixed_shapes is False
             else [self.max_mel_length, 80],
             "mel_lengths": [],
+            "real_mel_lengths": [],
             "g_attentions": [None, None]
             if self.use_fixed_shapes is False
             else [self.max_char_length, self.max_mel_length // self.reduction_factor],
@@ -243,6 +257,7 @@ def get_output_dtypes(self):
             "speaker_ids": tf.int32,
             "mel_gts": tf.float32,
             "mel_lengths": tf.int32,
+            "real_mel_lengths": tf.int32,
             "g_attentions": tf.float32,
         }
         return output_types

examples/tacotron2/train_tacotron2.py

@@ -368,6 +368,7 @@ def main():
         raise ValueError("Only npy are supported.")
 
     train_dataset = CharactorMelDataset(
+        dataset=config["tacotron2_params"]["dataset"],
         root_dir=args.train_dir,
         charactor_query=charactor_query,
         mel_query=mel_query,
@@ -394,6 +395,7 @@ def main():
     )
 
     valid_dataset = CharactorMelDataset(
+        dataset=config["tacotron2_params"]["dataset"],
         root_dir=args.dev_dir,
         charactor_query=charactor_query,
         mel_query=mel_query,