add pc-nsf training method

Author: yxlllc

configs/base_hifi.yaml

@@ -10,6 +10,10 @@ pe: 'parselmouth' # 'parselmouth' or 'harvest'
 f0_min: 65
 f0_max: 1100
 
+pc_aug: false # pc-nsf training method
+pc_aug_prob: 0.5
+pc_aug_key: 5
+
 aug_min: 0.9
 aug_max: 1.4
 aug_num: 1
@@ -34,15 +38,14 @@ valid_set_name: valid
 train_set_name: train
 
 
-volume_aug: True
+volume_aug: true
 volume_aug_prob: 0.5
 
 
 mel_vmin: -6. #-6.
 mel_vmax: 1.5
 
 
-mini_nsf: false
 audio_sample_rate: 44100
 audio_num_mel_bins: 128
 hop_size: 512            # Hop size.
@@ -62,6 +65,7 @@ crop_mel_frames: 20
 
 #model_cls: training.nsf_HiFigan_task.nsf_HiFigan
 model_args:
+  mini_nsf: false
   upsample_rates: [ 8, 8, 2, 2, 2 ]
   upsample_kernel_sizes: [ 16,16, 4, 4, 4 ]
   upsample_initial_channel: 512

configs/ft_hifigan.yaml

@@ -10,6 +10,10 @@ pe: 'parselmouth' # 'parselmouth' or 'harvest'
 f0_min: 65
 f0_max: 1100
 
+pc_aug: false # pc-nsf training method
+pc_aug_prob: 0.5
+pc_aug_key: 5
+
 aug_min: 0.9
 aug_max: 1.4
 aug_num: 1
@@ -42,7 +46,6 @@ mel_vmin: -6. #-6.
 mel_vmax: 1.5
 
 
-mini_nsf: false
 audio_sample_rate: 44100
 audio_num_mel_bins: 128
 hop_size: 512            # Hop size.
@@ -63,6 +66,7 @@ crop_mel_frames: 32
 
 #model_cls: training.nsf_HiFigan_task.nsf_HiFigan
 model_args:
+  mini_nsf: false
   upsample_rates: [ 8, 8, 2, 2, 2 ]
   upsample_kernel_sizes: [ 16,16, 4, 4, 4 ]
   upsample_initial_channel: 512

export_ckpt.py

@@ -47,9 +47,7 @@ def export(exp_name, ckpt_path, save_path, work_dir):
         new_config['win_size'] = config['win_size']
         new_config['fmin'] = config['fmin']
         new_config['fmax'] = config['fmax']
-        if 'mini_nsf' in config.keys():
-            new_config['mini_nsf'] = config['mini_nsf']
-        else:
+        if 'mini_nsf' not in new_config.keys():
             new_config['mini_nsf'] = False
         json_file.write(json.dumps(new_config, indent=1))
         print("Export configuration file successfully: ", new_config_file)

modules/loss/HiFiloss.py

@@ -29,26 +29,21 @@ def discriminator_loss(self, disc_real_outputs, disc_generated_outputs):
         loss = 0
         rlosses = 0
         glosses = 0
-        r_losses = []
-        g_losses = []
 
         for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
             r_loss = torch.mean((1 - dr) ** 2)
             g_loss = torch.mean(dg ** 2)
             loss += r_loss + g_loss
             rlosses += r_loss.item()
             glosses += g_loss.item()
-            r_losses.append(r_loss.item())
-            g_losses.append(g_loss.item())
 
-        return loss, rlosses, glosses, r_losses, g_losses
+        return loss, rlosses, glosses
 
     def Dloss(self, Dfake, Dtrue):
-
         (Fmsd_out, _), (Fmpd_out, _) = Dfake
         (Tmsd_out, _), (Tmpd_out, _) = Dtrue
-        msdloss, msdrlosses, msdglosses, _, _ = self.discriminator_loss(Tmsd_out, Fmsd_out)
-        mpdloss, mpdrlosses, mpdglosses, _, _ = self.discriminator_loss(Tmpd_out, Fmpd_out)
+        msdloss, msdrlosses, msdglosses = self.discriminator_loss(Tmsd_out, Fmsd_out)
+        mpdloss, mpdrlosses, mpdglosses = self.discriminator_loss(Tmpd_out, Fmpd_out)
         loss = msdloss + mpdloss
         return loss, {'DmsdlossF': msdglosses, 'DmsdlossT': msdrlosses, 'DmpdlossT': mpdrlosses,
                       'DmpdlossF': mpdglosses}
@@ -57,55 +52,42 @@ def feature_loss(self, fmap_r, fmap_g):
         loss = 0
         for dr, dg in zip(fmap_r, fmap_g):
             for rl, gl in zip(dr, dg):
-                loss += torch.mean(torch.abs(rl - gl))
-
+                b = min(rl.shape[0], gl.shape[0])
+                loss += torch.mean(torch.abs(rl[: b] - gl[: b]))
         return loss * 2
 
     def GDloss(self, GDfake, GDtrue):
         loss = 0
-        gen_losses = []
         msd_losses = 0
         mpd_losses = 0
+
         (msd_out, Fmsd_feature), (mpd_out, Fmpd_feature) = GDfake
         (_, Tmsd_feature), (_, Tmpd_feature) = GDtrue
+        
         for dg in msd_out:
-            l = torch.mean((1 - dg) ** 2)
-            gen_losses.append(l.item())
-            # loss += l
-            msd_losses = l + msd_losses
-
+            msd_losses += torch.mean((1 - dg) ** 2)
         for dg in mpd_out:
-            l = torch.mean((1 - dg) ** 2)
-            gen_losses.append(l.item())
-            # loss += l
-            mpd_losses = l + mpd_losses
-
+            mpd_losses += torch.mean((1 - dg) ** 2)
+        
         msd_feature_loss = self.feature_loss(Tmsd_feature, Fmsd_feature)
         mpd_feature_loss = self.feature_loss(Tmpd_feature, Fmpd_feature)
-        # loss +=msd_feature_loss
-        # loss +=mpd_feature_loss
+
         loss = msd_feature_loss + mpd_feature_loss + mpd_losses + msd_losses
-        # (msd_losses, mpd_losses), (msd_feature_loss, mpd_feature_loss), gen_losses
+
         return loss, {'Gmsdloss': msd_losses, 'Gmpdloss': mpd_losses, 'Gmsd_feature_loss': msd_feature_loss,
                       'Gmpd_feature_loss': mpd_feature_loss}
 
-    def Auxloss(self, Goutput, sample):
-        Gmel = self.mel.dynamic_range_compression_torch(self.mel(Goutput['audio'].squeeze(1)))
-        Rmel = self.mel.dynamic_range_compression_torch(self.mel(sample['audio'].squeeze(1)))
+    def Auxloss(self, Goutput, sample): 
+        Gwav = Goutput['audio'].squeeze(1)    
+        Rwav = sample['audio'].squeeze(1)
+        b = min(Gwav.shape[0], Rwav.shape[0])
+        Gmel = self.mel.dynamic_range_compression_torch(self.mel(Gwav[: b]))
+        Rmel = self.mel.dynamic_range_compression_torch(self.mel(Rwav[: b]))
         mel_loss = self.L1loss(Gmel, Rmel) * self.lab_aux_mel_loss
         if self.use_stftloss:
-            sc_loss, mag_loss = self.stft.stft(Goutput['audio'].squeeze(1), sample['audio'].squeeze(1))
+            sc_loss, mag_loss = self.stft.stft(Gwav[: b], Rwav[: b])
             stft_loss = (sc_loss + mag_loss) * self.lab_aux_stft_loss
             loss = mel_loss + stft_loss
-            return loss, {'auxloss': loss, 'auxloss_mel': mel_loss, 'auxloss_stft': stft_loss}
-        return mel_loss, {'auxloss': mel_loss}
-
-    # def Auxloss(self,Goutput, sample):
-    #
-    # Gmel=self.mel.dynamic_range_compression_torch(self.mel(Goutput['audio'].squeeze(1)))
-    # # Rmel=sample['mel']
-    # Rmel = self.mel.dynamic_range_compression_torch(self.mel(sample['audio'].squeeze(1)))
-    # sc_loss, mag_loss=self.stft.stft(Goutput['audio'].squeeze(1), sample['audio'].squeeze(1))
-    # loss=(sc_loss+ mag_loss)*self.labauxloss
-    # return loss,{'auxloss':loss,'auxloss_sc_loss':sc_loss,'auxloss_mag_loss':mag_loss}
-    #
+            return loss, {'aux_mel_loss': mel_loss, 'aux_stft_loss': stft_loss}
+        return mel_loss, {'aux_mel_loss': mel_loss}
+    

modules/loss/stft_loss.py

@@ -24,11 +24,9 @@ def stft(x, fft_size, hop_size, win_length, window):
 
     """
     x_stft = torch.stft(x, fft_size, hop_size, win_length, window, return_complex=True)
-    real = x_stft.real
-    imag = x_stft.imag
 
     # NOTE(kan-bayashi): clamp is needed to avoid nan or inf
-    return torch.sqrt(torch.clamp(real ** 2 + imag ** 2, min=1e-7)).transpose(2, 1)
+    return torch.clamp(x_stft.abs(), min=10**(-3.5)).transpose(2, 1)
 
 
 class SpectralConvergenceLoss(torch.nn.Module):
@@ -108,12 +106,10 @@ def forward(self, x, y):
 
 
 class warp_stft:
-    def __init__(self,cfg={},divce='cuda'):
-        self.stft=MultiResolutionSTFTLoss(**cfg).to(divce)
+    def __init__(self, cfg={}, device='cuda'):
+        self.stft = MultiResolutionSTFTLoss(**cfg).to(device)
 
-
-
-    def loss(self,x, y):
+    def loss(self, x, y):
         return self.stft(x, y)