Remove spacy model from toml (#1771)

* Remove spacy model from toml

* Semver

Author: AlonsoGuevara

.semversioner/next-release/patch-20250226045334122265.json

@@ -0,0 +1,4 @@
+{
+  "type": "patch",
+  "description": "Remove spacy model from toml file"
+}

graphrag/index/operations/build_noun_graph/np_extractors/base.py

@@ -3,8 +3,13 @@
 
 """Base class for noun phrase extractors."""
 
+import logging
 from abc import ABCMeta, abstractmethod
 
+import spacy
+
+log = logging.getLogger(__name__)
+
 
 class BaseNounPhraseExtractor(metaclass=ABCMeta):
     """Abstract base class for noun phrase extractors."""
@@ -37,3 +42,20 @@ def extract(self, text: str) -> list[str]:
     @abstractmethod
     def __str__(self) -> str:
         """Return string representation of the extractor, used for cache key generation."""
+
+    @staticmethod
+    def load_spacy_model(
+        model_name: str, exclude: list[str] | None = None
+    ) -> spacy.language.Language:
+        """Load a SpaCy model."""
+        if exclude is None:
+            exclude = []
+        try:
+            return spacy.load(model_name, exclude=exclude)
+        except OSError:
+            msg = f"Model `{model_name}` not found. Attempting to download..."
+            log.info(msg)
+            from spacy.cli.download import download
+
+            download(model_name)
+            return spacy.load(model_name, exclude=exclude)

graphrag/index/operations/build_noun_graph/np_extractors/cfg_extractor.py

@@ -5,7 +5,6 @@
 
 from typing import Any
 
-import spacy
 from spacy.tokens.doc import Doc
 
 from graphrag.index.operations.build_noun_graph.np_extractors.base import (
@@ -57,9 +56,13 @@ def __init__(
         self.include_named_entities = include_named_entities
         self.exclude_entity_tags = exclude_entity_tags
         if not include_named_entities:
-            self.nlp = spacy.load(model_name, exclude=["lemmatizer", "parser", "ner"])
+            self.nlp = self.load_spacy_model(
+                model_name, exclude=["lemmatizer", "parser", "ner"]
+            )
         else:
-            self.nlp = spacy.load(model_name, exclude=["lemmatizer", "parser"])
+            self.nlp = self.load_spacy_model(
+                model_name, exclude=["lemmatizer", "parser"]
+            )
 
         self.exclude_pos_tags = exclude_pos_tags
         self.noun_phrase_grammars = noun_phrase_grammars

graphrag/index/operations/build_noun_graph/np_extractors/syntactic_parsing_extractor.py

@@ -5,7 +5,6 @@
 
 from typing import Any
 
-import spacy
 from spacy.tokens.span import Span
 from spacy.util import filter_spans
 
@@ -55,9 +54,9 @@ def __init__(
         self.include_named_entities = include_named_entities
         self.exclude_entity_tags = exclude_entity_tags
         if not include_named_entities:
-            self.nlp = spacy.load(model_name, exclude=["lemmatizer", "ner"])
+            self.nlp = self.load_spacy_model(model_name, exclude=["lemmatizer", "ner"])
         else:
-            self.nlp = spacy.load(model_name, exclude=["lemmatizer"])
+            self.nlp = self.load_spacy_model(model_name, exclude=["lemmatizer"])
 
         self.exclude_pos_tags = exclude_pos_tags
 

poetry.lock

@@ -47,7 +47,6 @@ format-jinja = """
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
 environs = "^11.0.0"
-en-core-web-md = { url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.8.0/en_core_web_md-3.8.0.tar.gz" }
 
 # Vector Stores
 azure-search-documents = "^11.5.2"