remove rows with # from yodas + better logs

Author: AudranBert

ssak/utils/kaldi_converter.py

@@ -113,15 +113,17 @@ def merge_data(self, dataset, new_data):
                 diff_a_b = set(dict_dataset.keys()).difference(set(dict_new_data.keys()))
                 diff_b_a = set(dict_new_data.keys()).difference(set(dict_dataset.keys()))
                 logger.warning(f"The data you are trying to merge have different lengths at step {self.__class__.__name__} (execute_order={self.execute_order})!")
-                logger.warning(f"Dataset {len(dataset)} has {len(diff_a_b)} not present in new data")
-                logger.warning(f"New data {len(new_data)} has {len(diff_b_a)} not present in dataset")
-                logger.warning("Writing ids to debug.txt")
-                with open("debug.txt", "w") as f:
-                    if len(diff_a_b) > 0:
-                        f.write("In datset but not in new data:\n")
-                        for i in diff_a_b:
-                            f.write(f"{i}\n")
-                    if len(diff_b_a) > 0:
+                logger.warning(f"Dataset ({len(dataset)} rows) has {len(diff_a_b)} rows not present in new data")
+                logger.warning(f"New data ({len(new_data)} rows) has {len(diff_b_a)} rows not present in dataset")
+                logger.warning("Writing ids to log2kaldi/missing_ids.txt")
+                os.makedirs("log2kaldi", exist_ok=True)
+                if len(diff_a_b) > 0:
+                    with open(os.path.join("kaldi_data_processing",f"merge_new_data_missing_{self.execute_order}_{self.__class__.__name__}.txt"), "w") as f:
+                            f.write("In dataset but not in new data:\n")
+                            for i in diff_a_b:
+                                f.write(f"{i}\n")
+                if len(diff_b_a) > 0:
+                    with open(os.path.join("kaldi_data_processing",f"merge_dataset_missing_{self.execute_order}_{self.__class__.__name__}.txt"), "w") as f:
                         f.write("In new data but not in dataset:\n")
                         for i in diff_b_a:
                             f.write(f"{i}\n")

ssak/utils/kaldi_dataset.py

@@ -10,6 +10,7 @@
 
 logger = logging.getLogger(__name__)
 
+LOG_FOLDER = "kaldi_data_processing"
 
 @dataclass
 class KaldiDatasetRow:
@@ -266,7 +267,8 @@ def check_if_segments_in_audios(self, acceptance_end_s=0.25):
                 new_data.append(row)
         self.dataset = new_data
         logger.info(f"Removed {len(removed_lines)} segments that were not in audios (start or end after audio), check removed_lines_not_in_audios file")
-        with open("removed_lines_not_in_audios", "w") as f:
+        os.makedirs(LOG_FOLDER, exist_ok=True)
+        with open(os.path.join(LOG_FOLDER, "removed_lines_not_in_audios"), "w") as f:
             for row in removed_lines:
                 f.write(str(row) + "\n")
 
@@ -380,7 +382,8 @@ def normalize_audios(self, output_wavs_conversion_folder, target_sample_rate=160
                 else:
                     removed_lines.append(row)
             self.dataset = new_dataset
-            with open("removed_lines_audio_empty", "w") as f:
+            os.makedirs(LOG_FOLDER, exist_ok=True)
+            with open(os.path.join(LOG_FOLDER, "removed_lines_audio_empty"), "w") as f:
                 for row in removed_lines:
                     f.write(str(row) + "\n")
 
@@ -594,7 +597,8 @@ def apply_filter(self, filter, filter_out=True):
             else:
                 removed_lines.append(row)
         self.dataset = new_data
-        with open("filtered_out", "w") as f:
+        os.makedirs(LOG_FOLDER, exist_ok=True)
+        with open(os.path.join(LOG_FOLDER, "filtered_out"), "w") as f:
             for row in removed_lines:
                 f.write(str(row) + "\n")
 

tools/kaldi/datasets2kaldi/yodas2kaldi.py

@@ -13,7 +13,7 @@
     parser = argparse.ArgumentParser(description="Convert yodas dataset to Kaldi format")
     parser.add_argument("--force", action="store_true", default=True)
     parser.add_argument("--input", type=str, default="/data-server/datasets/audio/transcript/fr/YODAS/fr000")
-    parser.add_argument("--output", type=str, default="/data-server/datasets/audio/kaldi/fr/YODAS/fr000")
+    parser.add_argument("--output", type=str, default="/data-server/datasets/audio/kaldi/fr/YODAS/fr000_2")
     args = parser.parse_args()
 
     input_dataset = args.input
@@ -52,14 +52,18 @@
     spk_ids = Row2Info("id", ["speaker"], 4, None, None)
     dev_reader = Reader2Kaldi(input_dataset, processors=[texts, durations, audios, audio_ids, spk_ids])
     dataset = dev_reader.load(debug=False, accept_missing_speaker=True)
-    dataset.normalize_audios(os.path.join(input_dataset, "converted"), target_extension="wav", num_workers=16)
-
+    
     def filter(row):
         if row.id.startswith("E--pPwqi_50-"):
             return True
+        elif "#" in row.text:
+            return True
         return False
 
     removed_lines = dataset.apply_filter(filter)
+    
+    dataset.normalize_audios(os.path.join(input_dataset, "converted"), target_extension="wav", num_workers=16)
+
     logger.info(f"Dataset duration: {dataset.get_duration('sum')/3600:.2f}h")
     dataset.save(raw, False)