update hybrid data

Author: Atticus1806

common/baselines/tedlium2/hybrid/data.py

@@ -26,7 +26,7 @@ def build_hdf_data_input(
     partition_epoch: int = 1,
     acoustic_mixtures: Optional = None,
     seq_ordering: str = "sorted",
-):
+) -> HdfDataInput:
     """
     Dumps features and alignments from RASR into hdfs, to enable full RETURNN training
     :param features: Feature bundle generated by the dump_features_for_hybrid_training function
@@ -37,7 +37,7 @@ def build_hdf_data_input(
     :param partition_epoch: Partition epoch for the alignment dataset, mainly relevant for training dataset
     :param acoustic_mixtures: Acoustic mixture file from the GMM for prior calculation, most likely going to be replaced
     :param seq_ordering: sequence ordering for the align dataset, usually sorted for dev/eval and laplace for train
-    :return:
+    :return: HdfDataInput with corresponding hdf datasets
     """
 
     feat_dataset = {
@@ -56,14 +56,11 @@ def build_hdf_data_input(
         "seq_list_filter_file": segment_list,
     }
 
-    feat_job = ReturnnDumpHDFJob(
-        data=feat_dataset,
-        returnn_python_exe=RETURNN_EXE,
-        returnn_root=RETURNN_RC_ROOT,
-    )
+    feat_job = ReturnnDumpHDFJob(data=feat_dataset, returnn_python_exe=RETURNN_EXE, returnn_root=RETURNN_RC_ROOT)
     if alias_prefix is not None:
         feat_job.add_alias(alias_prefix + "/dump_features")
     feat_hdf = feat_job.out_hdf
+
     align_dataset = {
         "class": "SprintCacheDataset",
         "data": {
@@ -104,7 +101,7 @@ def dump_features_for_hybrid_training(
     :param gmm_system: GMM system to get corpora from
     :param feature_extraction_args: Args for the feature extraction
     :param feature_extraction_class: Feature extraction class/job to be used for extraction
-    :return:
+    :return: path to the train cv and devtrain features
     """
     features = {}
     for name in ["nn-train", "nn-cv", "nn-devtrain"]:
@@ -119,7 +116,6 @@ def get_corpus_data_inputs(
     feature_extraction_args: Dict[str, Any],
     feature_extraction_class: Callable[[Any], FeatureExtractionJob],
     alias_prefix: Optional[str] = None,
-    remove_faulty_segments: bool = False,
 ) -> Tuple[
     Dict[str, HdfDataInput],
     Dict[str, HdfDataInput],
@@ -134,7 +130,7 @@ def get_corpus_data_inputs(
     :param feature_extraction_args: Args for the feature extraction of the hybrid features (might be different from GMM)
     :param feature_extraction_class: Feature extraction class/job to be used for extraction
     :param alias_prefix: Prefix for naming of experiments
-    :return:
+    :return: HdfDataInputs for the train sets and ReturnnRasrDataInputs for the dev and train sets
     """
 
     train_corpus_path = gmm_system.corpora["train"].corpus_file
@@ -147,11 +143,6 @@ def get_corpus_data_inputs(
     total_train_num_segments = NUM_SEGMENTS["train"]
 
     all_train_segments = corpus_recipe.SegmentCorpusJob(train_corpus_path, 1).out_single_segment_files[1]
-    if remove_faulty_segments:
-        all_train_segments = corpus_recipe.FilterSegmentsByListJob(
-            segment_files={1: all_train_segments},
-            filter_list=["TED-LIUM-realease2/AndrewMcAfee_2013/23", "TED-LIUM-realease2/iOTillettWright_2012X/43"],
-        ).out_single_segment_files[1]
     cv_segments = corpus_recipe.SegmentCorpusJob(cv_corpus_path, 1).out_single_segment_files[1]
 
     dev_train_size = 500 / total_train_num_segments
@@ -213,7 +204,7 @@ def get_corpus_data_inputs(
         allophone_labeling=allophone_labeling,
         alias_prefix=alias_prefix + "/nn_train_data",
         partition_epoch=5,
-        acoustic_mixtures=gmm_system.outputs["train"]["final"].acoustic_mixtures,  # TODO: NN Mixtures
+        acoustic_mixtures=gmm_system.outputs["train"]["final"].acoustic_mixtures,
         seq_ordering="laplace:.1000",
     )
     tk.register_output(f"{alias_prefix}/nn_train_data/features", nn_train_data.features)