bug(medcat): CU-869b2hpam Fix issue loading tokenizers off disk (#213)

* CU-869b2hpam: Make tokenizer init use model load path to amend saved tokenizer path

* CU-869b2hpam: Fix issue with lang string from tokenizer folder name when setting stopwords

* CU-869b2hpam: Make tokenizer saving return subfolder name only (not path to it)

* CU-869b2hpam: Minor whitespace changes

* CU-869b2hpam: Add a logged warning when fixing the spacy model internals path

* CU-869b2hpam: Fix tests regarding spacy model save

* CU-869b2hpam: Fix tests regarding config merge

Author: mart-r

medcat-v2/medcat/pipeline/pipeline.py

@@ -79,7 +79,7 @@ def __init__(self, cdb: CDB, vocab: Optional[Vocab],
         #       but it should be non-None otherwise
         self.vocab: Vocab = vocab  # type: ignore
         self.config = self.cdb.config
-        self._tokenizer = self._init_tokenizer()
+        self._tokenizer = self._init_tokenizer(model_load_path)
         self._components: list[CoreComponent] = []
         self._addons: list[AddonComponent] = []
         self._init_components(model_load_path, old_pipe, addon_config_dict)
@@ -95,8 +95,20 @@ def tokenizer_with_tag(self) -> BaseTokenizer:
         tag_comp = self.get_component(CoreComponentType.tagging)
         return DelegatingTokenizer(self.tokenizer, [tag_comp])
 
-    def _init_tokenizer(self) -> BaseTokenizer:
+    def _init_tokenizer(self, model_load_path: Optional[str]) -> BaseTokenizer:
         nlp_cnf = self.config.general.nlp
+        if model_load_path:
+            orig_modelname = nlp_cnf.modelname
+            model_basename = os.path.basename(orig_modelname)
+            # NOTE: this should update the load path to the correct one
+            nlp_cnf.modelname = os.path.join(
+                model_load_path, model_basename)
+            if orig_modelname != model_basename:
+                logger.warning(
+                    "Loading a model with incorrectly saved tokenizer "
+                    "internals path. Was saved as '%s' whereas should have "
+                    "had just '%s'. This is an automated fix - no further "
+                    "action is needed", orig_modelname, model_basename)
         try:
             return create_tokenizer(nlp_cnf.provider, self.config)
         except TypeError as type_error:

medcat-v2/medcat/tokenizing/spacy_impl/tokenizers.py

@@ -58,7 +58,8 @@ def __init__(self, spacy_model_name: str,
             ensure_spacy_model(self._spacy_model_name)
             spacy_model_name = self._spacy_model_name
         if stopwords is not None:
-            lang_str = os.path.basename(spacy_model_name).split('_', 1)[0]
+            lang_str = os.path.basename(spacy_model_name).removeprefix(
+                TOKENIZER_PREFIX).split('_', 1)[0]
             cls = spacy.util.get_lang_class(lang_str)
             cls.Defaults.stop_words = set(stopwords)
         self._nlp = spacy.load(spacy_model_name,
@@ -87,7 +88,8 @@ def __call__(self, text: str) -> MutableDocument:
     @classmethod
     def create_new_tokenizer(cls, config: Config) -> 'SpacyTokenizer':
         nlp_cnf = config.general.nlp
-        return cls(nlp_cnf.modelname,
+        return cls(
+            nlp_cnf.modelname,
             nlp_cnf.disabled_components,
             config.general.diacritics,
             config.preprocessing.max_document_length,
@@ -102,14 +104,14 @@ def get_entity_class(self) -> Type[MutableEntity]:
     # saveable tokenizer
 
     def save_internals_to(self, folder_path: str) -> str:
-        subfolder = os.path.join(
-            folder_path, f"{TOKENIZER_PREFIX}{self._spacy_model_name}")
+        subfolder_only = f"{TOKENIZER_PREFIX}{self._spacy_model_name}"
+        subfolder = os.path.join(folder_path, subfolder_only)
         if os.path.exists(subfolder):
             # NOTE: always overwrite
             shutil.rmtree(subfolder)
         logger.debug("Saving spacy model to '%s'", subfolder)
         self._nlp.to_disk(subfolder)
-        return subfolder
+        return subfolder_only
 
     def load_internals_from(self, folder_path: str) -> bool:
         return os.path.exists(folder_path)

medcat-v2/tests/test_cat.py

@@ -129,8 +129,9 @@ class ConfigMergeTests(unittest.TestCase):
     def test_can_merge_config(self):
         model = cat.CAT.load_model_pack(
             EXAMPLE_MODEL_PACK_ZIP, config_dict=self.model_dict)
-        self.assertEqual(
-            model.config.general.nlp.modelname, self.spacy_model_name)
+        # NOTE: this is converted to a (non-existent) path
+        self.assertIn(
+            self.spacy_model_name, model.config.general.nlp.modelname)
 
 
 class OntologiesMapTests(TrainedModelTests):
@@ -919,7 +920,9 @@ def setUpClass(cls):
         cls.saved_model_path = cls.cat.save_model_pack(
             cls._save_folder.name, make_archive=False)
         # NOTE: that has changed config
-        cls.saved_spacy_path = cls.cat.config.general.nlp.modelname
+        cls.saved_spacy_path = os.path.join(
+            cls.saved_model_path,
+            cls.cat.config.general.nlp.modelname)
 
     @classmethod
     def tearDownClass(cls):