fix: model count

Author: VinciGit00

examples/local_models/smart_scraper_ollama.py

@@ -9,7 +9,7 @@
 
 graph_config = {
     "llm": {
-        "model": "ollama/llama3.1",
+        "model": "ollama/mistral",
         "temperature": 0,
         "format": "json",  # Ollama needs the format to be specified explicitly
         # "base_url": "http://localhost:11434", # set ollama URL arbitrarily

scrapegraphai/graphs/abstract_graph.py

@@ -149,7 +149,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
 
         known_models = ["openai", "azure_openai", "google_genai", "ollama", "oneapi", "nvidia", "groq", "google_vertexai", "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", "fireworks"]
 
-        if llm_params["model"] not in known_models:
+        if llm_params["model"].split("/")[0] not in known_models:
             raise ValueError(f"Model '{llm_params['model']}' is not supported")
 
         try:

scrapegraphai/helpers/models_tokens.py

@@ -62,6 +62,7 @@
         "scrapegraph": 8192, 
         "llava": 4096, 
         "mixtral:8x22b-instruct": 65536, 
+        "mistral":8192,
         "mistral-openorca": 32000, 
         "nomic-embed-text": 8192, 
         "nous-hermes2:34b": 4096, 

scrapegraphai/nodes/parse_node.py

@@ -80,6 +80,7 @@ def execute(self, state: dict) -> dict:
             docs_transformed = docs_transformed[0]
 
             if isinstance(docs_transformed, Document):
+                
                 chunks = chunk(text=docs_transformed.page_content,
                             chunk_size=self.node_config.get("chunk_size", 4096)-250,
                             token_counter=lambda text: len(text.split()),