add more checks and filtering in nemo pipeline

Author: AudranBert

ssak/utils/kaldi_dataset.py

@@ -247,6 +247,29 @@ def get_duration(self, mode=sum, target="segment"):
             return mode(durations)
         return mode([i.duration for i in self.dataset])
 
+    def check_if_segments_in_audios(self, acceptance_end_s=0.25):
+        from pydub.utils import mediainfo
+
+        new_data = []
+        removed_lines = []
+        files_duration = dict()
+        for row in tqdm(self, desc="Check if segments are in audios"):
+            if row.audio_path not in files_duration:
+                dur = round(float(mediainfo(row.audio_path)["duration"]), 3)
+                files_duration[row.audio_path] = dur
+            dur = files_duration[row.audio_path]
+            if row.start >= dur:
+                removed_lines.append(row)
+            elif row.end > dur + acceptance_end_s:
+                removed_lines.append(row)
+            else:
+                new_data.append(row)
+        self.dataset = new_data
+        logger.info(f"Removed {len(removed_lines)} segments that were not in audios (start or end after audio), check removed_lines_not_in_audios file")
+        with open("removed_lines_not_in_audios", "w") as f:
+            for row in removed_lines:
+                f.write(str(row) + "\n")
+
     def filter_by_audio_ids(self, audio_ids):
         """
         Filter the dataset by audio ids
@@ -357,7 +380,7 @@ def normalize_audios(self, output_wavs_conversion_folder, target_sample_rate=160
                 else:
                     removed_lines.append(row)
             self.dataset = new_dataset
-            with open("removed_lines", "w") as f:
+            with open("removed_lines_audio_empty", "w") as f:
                 for row in removed_lines:
                     f.write(str(row) + "\n")
 

tools/nemo/convert_kaldi_dataset_to_nemo.py

@@ -3,6 +3,7 @@
 import logging
 import os
 
+from find_long_transcriptions import filter_incoherent_segments
 from tqdm import tqdm
 
 from ssak.utils.kaldi_dataset import KaldiDataset
@@ -49,7 +50,7 @@ def kaldi_to_nemo(kaldi_dataset, output_file):
             f.write("\n")
 
 
-def convert_dataset(kaldi_input_dataset, output_dir, new_audio_folder=None, check_audio=False):
+def convert_dataset(kaldi_input_dataset, output_dir, new_audio_folder=None, check_audio=False, check_if_in_audio=False, remove_incoherent_texts=False):
     logger.info(f"Converting Kaldi dataset {kaldi_input_dataset} to NeMo format")
     splitted_path = kaldi_input_dataset.split(os.sep)
     if splitted_path[-1] == "":
@@ -82,9 +83,15 @@ def convert_dataset(kaldi_input_dataset, output_dir, new_audio_folder=None, chec
             target_extension="wav",
             num_workers=6,
         )  # wavs are faster to load than mp3
+    if check_if_in_audio:
+        logger.info("Check if segments are in audios")
+        kaldi_dataset.check_if_segments_in_audios()
     logger.info(f"Writing to {file}")
     os.makedirs(output_dir, exist_ok=True)
     kaldi_to_nemo(kaldi_dataset, file)
+    if remove_incoherent_texts:
+        logger.info("Check for incoherent texts (very long text with a short audio segment)")
+        filter_incoherent_segments(file, file + "_removed_lines")
     logger.info(f"Conversion done (saved to {len(kaldi_dataset)} lines to {file})")
 
 

tools/nemo/convert_kaldi_datasets_to_nemo.py

@@ -9,7 +9,7 @@
 logger = logging.getLogger(__name__)
 
 
-def convert_datasets(inputs: list, output_file, output_wav_dir=None, check_audio=False):
+def convert_datasets(inputs: list, output_file, output_wav_dir=None, check_audio=False, check_if_in_audio=False, remove_incoherent_texts=False):
     input_files = inputs
     if len(input_files) == 1:
         logger.warning("One input file, considering it as containing a list of files")
@@ -20,7 +20,7 @@ def convert_datasets(inputs: list, output_file, output_wav_dir=None, check_audio
             raise FileNotFoundError(f"Non-existing file {input_folder}")
         if not os.path.isdir(input_folder):
             raise NotADirectoryError(f"File {input_folder} is not a directory")
-        convert_dataset(input_folder, output_file, output_wav_dir, check_audio=check_audio)
+        convert_dataset(input_folder, output_file, output_wav_dir, check_audio=check_audio, check_if_in_audio=check_if_in_audio, remove_incoherent_texts=remove_incoherent_texts)
     logger.info(f"Finished converting datasets from {input_files} to {output_file}")
 
 

tools/nemo/find_long_transcriptions.py

@@ -0,0 +1,63 @@
+import argparse
+import json
+import logging
+import shutil
+
+from tqdm import tqdm
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# En règle générale, les conversations entre adulte se font à un débit de 200 mots/minute.
+# Certains montent la cadence jusqu’à 300 mots/minute. Alors que pour des enregistrements audio par exemple, on préconise un rythme de 150 mots/minute pour être bien audible.
+# a bit less than 5 characters per word, 4,65 for journalistic, and 4,89 for litterature
+# if we add 2 more characters (spaces) it means we have around 7 characters per word (8 to be large)
+# fast reading person: 160wpm (2.7wps)
+# speaking person: 200wpm (3.3wps)
+# ultra fast: 300wpm (5wps) (probably possible to happen for very short duration but not for more than a few sec)
+# so for ultra fast 1s we have around 5*8=40 characters
+# So, we can safely say that we it can't go over 50
+# 5s: 3.3*8*5=132c/ 5*8*5=200
+# 10s: 3.3*8*10=264c/ 400c
+# 20s: 528c
+# 30s: 792c
+
+INCOHERENT_THREEHOLD = {1: 50, 5: 200, 10: 350, 20: 550, 30: 700}
+
+
+def filter_incoherent_segments(input_file, filtered_out_file):
+    with open(input_file, encoding="utf-8") as f:
+        lines = f.readlines()
+        data = [json.loads(l) for l in lines]
+    ct_dict = {i: 0 for i in list(INCOHERENT_THREEHOLD.values())}
+    ct = 0
+    with open(input_file + ".tmp", "w", encoding="utf-8") as f, open(filtered_out_file, "w", encoding="utf-8") as log:
+        for i, row in enumerate(tqdm(data, desc="Checking for incoherent texts lengths")):
+            dur = float(row["duration"])
+            max_text = None
+            for k, v in INCOHERENT_THREEHOLD.items():
+                if dur < k:
+                    max_text = v
+                    break
+            if max_text is None:
+                max_text = list(INCOHERENT_THREEHOLD.values())[-1]
+            if len(row["text"]) > max_text:
+                ct += 1
+                ct_dict[max_text] = ct_dict[max_text] + 1
+                json.dump(row, log, ensure_ascii=False)
+                log.write("\n")
+            else:
+                json.dump(row, f, ensure_ascii=False)
+                f.write("\n")
+    print(f"Find {ct} long texts in {input_file}")
+    print(f"Removed: {ct_dict}")
+    shutil.move(input_file + ".tmp", input_file)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Remove incoherent lines from nemo manifest")
+    parser.add_argument("file", help="Input file", type=str)
+    parser.add_argument("output", help="output file", type=str)
+    # parser.add_argument('--max_char', help="Depends on segments max length", type=int, default=700)
+    args = parser.parse_args()
+    filter_incoherent_segments(args.file, args.output)

tools/nemo/pipeline_prepare_nemo_data.py

@@ -7,6 +7,10 @@
 from generate_dataset_list_files import generate_dataset_list_files
 from merge_manifest import merge_manifests
 
+CHECK_AUDIO = True
+CHECK_IF_SEGMENT_IN_AUDIO = False
+REMOVE_INCOHERENT_TEXTS = True
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Prepare data for Nemo")
     parser.add_argument("--train_input_datasets", help="Input datasets", type=str, default=None)
@@ -103,7 +107,9 @@
                 [os.path.join(tmp_manifest_dir, "datasets_list", f"{i}_datasets")],
                 os.path.join(tmp_manifest_dir, f"{i}_manifests"),
                 output_wav_dir,
-                check_audio=True,
+                check_audio=CHECK_AUDIO,
+                check_if_in_audio=CHECK_IF_SEGMENT_IN_AUDIO,
+                remove_incoherent_texts=REMOVE_INCOHERENT_TEXTS,
             )
         except FileExistsError:
             pass