nemo pipeline support from json input

Author: AudranBert

tools/nemo/generate_dataset_list_files.py

@@ -1,4 +1,5 @@
 import argparse
+import json
 import logging
 import os
 import re
@@ -13,9 +14,21 @@ def generate_dataset_list_files(dataset_list, dataset_folder, dest, mode, subset
         with open(dest) as f:
             return f.read().strip().split("\n")
     new_list = []
-    with open(dataset_list) as f:
-        datasets = f.read().strip().split("\n")
-
+    if dataset_list.endswith(".json"):
+        with open(dataset_list) as f:
+            data = json.load(f)
+        datasets = []
+        for d in data:
+            if data[d]:
+                if mode == "dev" and data[d].get("valid", False):
+                    datasets.append(data[d].get("kaldi_subpath", d))
+                elif mode == "train":
+                    datasets.append(data[d].get("kaldi_subpath", d))
+            elif mode == "train":
+                datasets.append(d)
+    else:
+        with open(dataset_list) as f:
+            datasets = f.read().strip().split("\n")
     patterns = ""
     if mode == "train":
         patterns = r"train$|split\d$"