add subset arg to pipeline

Author: AudranBert

tools/nemo/generate_dataset_list_files.py

@@ -7,13 +7,13 @@
 logger = logging.getLogger(__name__)
 
 
-def generate_dataset_list_files(dateset_list, dataset_folder, dest, mode, subset_pattern):
+def generate_dataset_list_files(dataset_list, dataset_folder, dest, mode, subset_pattern):
     if os.path.exists(dest):
         logger.info(f"Reading dataset list from {dest} (already exists)")
         with open(dest) as f:
             return f.read().strip().split("\n")
     new_list = []
-    with open(dateset_list) as f:
+    with open(dataset_list) as f:
         datasets = f.read().strip().split("\n")
 
     patterns = ""

tools/nemo/pipeline_prepare_nemo_data.py

@@ -19,6 +19,7 @@
     parser.add_argument("--datasets_folder", help="Dataset folder", type=str, default=None)
     parser.add_argument("--output_wav_dir", help="Output wav directory", type=str, default="processed_dataset")
     parser.add_argument("--manifest_dir", default="input_manifests")
+    parser.add_argument("--subset_pattern", default="nocasepunc_max30", type=str)
     # Options for creating a tokenizer using all splits
     parser.add_argument("--create_tokenizer", default=None, help="Folder to save tokenizer (if not set, no tokenizer is created)")
     parser.add_argument("--vocab_size", help="Vocab size", type=int, default=1024)
@@ -78,7 +79,7 @@
             datasets_folder,
             dest=os.path.join(tmp_manifest_dir, "datasets_list", "train_datasets"),
             mode="train",
-            subset_pattern="nocasepunc_max30",
+            subset_pattern=args.subset_pattern,
         )
     if args.test_input_datasets:
         splits_to_process.append("test")
@@ -87,7 +88,7 @@
             datasets_folder,
             dest=os.path.join(tmp_manifest_dir, "datasets_list", "test_datasets"),
             mode="test",
-            subset_pattern="nocasepunc_max30",
+            subset_pattern=args.subset_pattern,
         )
     if args.dev_input_datasets:
         splits_to_process.append("dev")
@@ -96,7 +97,7 @@
             datasets_folder,
             dest=os.path.join(tmp_manifest_dir, "datasets_list", "dev_datasets"),
             mode="dev",
-            subset_pattern="nocasepunc_max30",
+            subset_pattern=args.subset_pattern,
         )
     if len(splits_to_process) == 0:
         raise ValueError("No splits to process")