Merge pull request #10 from megagonlabs/spacy_v3.4

migrate to spacy v3.4

Author: hiroshi-matsuda-rit

ginza_transformers/layers/hf_shim_custom.py

@@ -124,11 +124,14 @@ def from_bytes(self, bytes_data):
                 transformer.load_state_dict(torch.load(filelike, map_location=map_location))
             else:
                 try:
-                    transformer = AutoModel.from_pretrained(config._name_or_path, local_files_only=True)
-                except OSError as e2:
-                    print("trying to download model from huggingface hub:", config._name_or_path, "...", file=sys.stderr)
-                    transformer = AutoModel.from_pretrained(config._name_or_path)
-                    print("succeded", file=sys.stderr)
+                    transformer = AutoModel.from_pretrained(config_dict["_name_or_path"], local_files_only=True)
+                except OSError as e1:
+                    try:
+                        transformer = AutoModel.from_pretrained(config._name_or_path)
+                    except OSError as e2:
+                        print("trying to download model from huggingface hub:", config_dict["_name_or_path"], "...", file=sys.stderr)
+                        transformer = AutoModel.from_pretrained(config_dict["_name_or_path"])
+                        print("succeded", file=sys.stderr)
 
             transformer.to(map_location)
             self._model = transformer

ginza_transformers/layers/transformer_model.py

@@ -55,10 +55,10 @@ def huggingface_from_pretrained_custom(
     try:
         trf_config["return_dict"] = True
         config = AutoConfig.from_pretrained(str_path, **trf_config)
-        transformer = AutoModel.from_pretrained(str_path, config=config)
-    except OSError as e:
+        transformer = AutoModel.from_pretrained(model_name, local_files_only=True)
+    except OSError as e1:
         try:
-            transformer = AutoModel.from_pretrained(str_path, local_files_only=True)
+            transformer = AutoModel.from_pretrained(str_path, config=config)
         except OSError as e2:
             model_name = str(source)
             print("trying to download model from huggingface hub:", model_name, "...", file=sys.stderr)

setup.py

@@ -12,11 +12,10 @@
     },
     install_requires=[
         "spacy-transformers>=1.1.2,<1.2.0",
-        "transformers<4.13.0",
     ],
     license="MIT",
     name="ginza-transformers",
     packages=find_packages(include=["ginza_transformers", "ginza_transformers.layers"]),
     url="https://github.com/megagonlabs/ginza-transformers",
-    version='0.4.1',
+    version='0.4.2',
 )