using tedlium v2

Author: Marvin84

common/baselines/tedlium2_v2/__init__.py

@@ -0,0 +1,42 @@
+from collections import defaultdict
+from typing import Dict
+
+from i6_experiments.common.datasets.tedlium2.constants import CONCURRENT
+from i6_experiments.common.datasets.tedlium2_v2.corpus import get_corpus_object_dict
+from i6_experiments.common.datasets.tedlium2_v2.lexicon import (
+    get_g2p_augmented_bliss_lexicon,
+)
+from i6_experiments.common.setups.rasr.util import RasrDataInput
+
+from i6_experiments.common.setups.rasr.config.lex_config import (
+    LexiconRasrConfig,
+)
+from i6_experiments.common.setups.rasr.config.lm_config import ArpaLmRasrConfig
+from i6_experiments.common.baselines.tedlium2_v2.lm.ngram_config import run_tedlium2_ngram_lm
+
+
+def get_corpus_data_inputs(add_unknown_phoneme_and_mapping: bool = False) -> Dict[str, Dict[str, RasrDataInput]]:
+    corpus_object_dict = get_corpus_object_dict(audio_format="wav", output_prefix="corpora")
+
+    train_lexicon = LexiconRasrConfig(
+        get_g2p_augmented_bliss_lexicon(
+            add_unknown_phoneme_and_mapping=add_unknown_phoneme_and_mapping, output_prefix="lexicon"
+        ),
+        False,
+    )
+
+    lms_system = run_tedlium2_ngram_lm(add_unknown_phoneme_and_mapping=add_unknown_phoneme_and_mapping)
+    lm = lms_system.interpolated_lms["dev-pruned"]["4gram"]
+    comb_lm = ArpaLmRasrConfig(lm_path=lm.ngram_lm)
+
+    rasr_data_input_dict = defaultdict(dict)
+
+    for name, crp_obj in corpus_object_dict.items():
+        rasr_data_input_dict[name][name] = RasrDataInput(
+            corpus_object=crp_obj,
+            lexicon=train_lexicon.get_dict(),
+            concurrent=CONCURRENT[name],
+            lm=comb_lm.get_dict() if name == "dev" or name == "test" else None,
+        )
+
+    return rasr_data_input_dict

common/baselines/tedlium2_v2/data.py

@@ -0,0 +1,27 @@
+"""
+List of default tools and software to be defined as default independent from hashing
+by setting one explicit hash.
+
+In order to use different software paths without hash changes, just use the same explicit hash string as given here.
+
+If you want a stronger guarantee that you get the intended results, please consider using the explicit software
+version listed here. Nevertheless, the most recent "head" should be safe to be used as well.
+"""
+from sisyphus import tk
+from i6_experiments.common.tools.audio import compile_ffmpeg_binary
+from i6_experiments.common.tools.rasr import compile_rasr_binaries_i6mode
+from i6_experiments.common.tools.sctk import compile_sctk
+
+RASR_BINARY_PATH = compile_rasr_binaries_i6mode(
+    branch="apptainer_tf_2_8", configure_options=["--apptainer-patch=2023-05-08_tensorflow-2.8_v1"]
+)  #  use most recent RASR
+# RASR_BINARY_PATH = tk.Path("/work/asr4/rossenbach/neon_test/rasr_versions/rasr_no_tf/arch/linux-x86_64-standard/")
+assert RASR_BINARY_PATH, "Please set a specific RASR_BINARY_PATH before running the pipeline"
+RASR_BINARY_PATH.hash_overwrite = "TEDLIUM2_DEFAULT_RASR_BINARY_PATH"
+
+
+SCTK_BINARY_PATH = compile_sctk(branch="v2.4.12")  # use last published version
+SCTK_BINARY_PATH.hash_overwrite = "TEDLIUM2_DEFAULT_SCTK_BINARY_PATH"
+
+SRILM_PATH = tk.Path("/work/tools/users/luescher/srilm-1.7.3/bin/i686-m64/")
+SRILM_PATH.hash_overwrite = "TEDLIUM2_DEFAULT_SRILM_PATH"