Add option to generate LM image and GC via two separate jobs

recognition/advanced_tree_search.py

@@ -10,6 +10,7 @@
 
 Path = setup_path(__package__)
 
+import copy
 import math
 import os
 import shutil
@@ -167,6 +168,7 @@ def __init__(
         lmgc_mem: float = 12.0,
         lmgc_alias: Optional[str] = None,
         lmgc_scorer: Optional[rasr.FeatureScorer] = None,
+        separate_lmi_gc_generation: bool = False,
         model_combination_config: Optional[rasr.RasrConfig] = None,
         model_combination_post_config: Optional[rasr.RasrConfig] = None,
         extra_config: Optional[rasr.RasrConfig] = None,
@@ -190,6 +192,7 @@ def __init__(
         :param lmgc_mem: Memory requirement for the AdvancedTreeSearchLmImageAndGlobalCacheJob
         :param lmgc_alias: Alias for the AdvancedTreeSearchLmImageAndGlobalCacheJob
         :param lmgc_scorer: Dummy scorer for the AdvancedTreeSearchLmImageAndGlobalCacheJob which is required but unused
+        :param separate_lmi_gc_generation: Whether to generate the LM image and the global cache via two separate jobs for a more stable hash
         :param model_combination_config: Configuration for model combination
         :param model_combination_post_config: Post config for model combination
         :param extra_config: Additional Config for recognition
@@ -286,18 +289,41 @@ def create_config(
         lmgc_mem: float,
         lmgc_alias: Optional[str],
         lmgc_scorer: Optional[rasr.FeatureScorer],
+        separate_lmi_gc_generation: bool,
         model_combination_config: Optional[rasr.RasrConfig],
         model_combination_post_config: Optional[rasr.RasrConfig],
         extra_config: Optional[rasr.RasrConfig],
         extra_post_config: Optional[rasr.RasrConfig],
         **kwargs,
     ):
-        lm_gc = AdvancedTreeSearchLmImageAndGlobalCacheJob(
-            crp, lmgc_scorer if lmgc_scorer is not None else feature_scorer, extra_config, extra_post_config
-        )
-        if lmgc_alias is not None:
-            lm_gc.add_alias(lmgc_alias)
-        lm_gc.rqmt["mem"] = lmgc_mem
+        def specialize_lm_config(crp, lm_config):
+            crp = copy.deepcopy(crp)
+            crp.language_model = lm_config
+            return crp
+
+        if separate_lmi_gc_generation:
+            gc = BuildGlobalCacheJob(crp, extra_config, extra_post_config).out_global_cache
+
+            arpa_lms = AdvancedTreeSearchLmImageAndGlobalCacheJob.find_arpa_lms(
+                crp.language_model, post_config.lm if post_config is not None else None
+            )
+            lm_images = {
+                (i + 1): lm.CreateLmImageJob(
+                    specialize_lm_config(crp, lm), extra_config=extra_config, extra_post_config=extra_post_config
+                ).out_lm
+                for i, lm in enumerate(arpa_lms)
+            }
+            lm_gc = None
+        else:
+            lm_gc = AdvancedTreeSearchLmImageAndGlobalCacheJob(
+                crp, lmgc_scorer if lmgc_scorer is not None else feature_scorer, extra_config, extra_post_config
+            )
+            if lmgc_alias is not None:
+                lm_gc.add_alias(lmgc_alias)
+            lm_gc.rqmt["mem"] = lmgc_mem
+
+            gc = lm_gc.out_global_cache
+            lm_images = lm_gc.out_lm_images
 
         search_parameters = cls.update_search_parameters(search_parameters)
 
@@ -397,14 +423,14 @@ def create_config(
             ]
 
         post_config.flf_lattice_tool.global_cache.read_only = True
-        post_config.flf_lattice_tool.global_cache.file = lm_gc.out_global_cache
+        post_config.flf_lattice_tool.global_cache.file = gc
 
         arpa_lms = AdvancedTreeSearchLmImageAndGlobalCacheJob.find_arpa_lms(
             config.flf_lattice_tool.network.recognizer.lm,
             post_config.flf_lattice_tool.network.recognizer.lm,
         )
         for i, lm_config in enumerate(arpa_lms):
-            lm_config[1].image = lm_gc.out_lm_images[i + 1]
+            lm_config[1].image = lm_images[i + 1]
 
         # Remaining Flf-network