Merge pull request #348 from tindo2003/fix_robots_node

Author: VinciGit00

scrapegraphai/nodes/robots_node.py

@@ -9,7 +9,6 @@
 from langchain.prompts import PromptTemplate
 from langchain.output_parsers import CommaSeparatedListOutputParser
 
-from .base_node import BaseNode
 from langchain.output_parsers import CommaSeparatedListOutputParser
 from langchain.prompts import PromptTemplate
 from langchain_community.document_loaders import AsyncChromiumLoader
@@ -18,7 +17,6 @@
 from ..utils.logging import get_logger
 from .base_node import BaseNode
 
-
 class RobotsNode(BaseNode):
     """
     A node responsible for checking if a website is scrapeable or not based on the robots.txt file.
@@ -48,13 +46,14 @@ def __init__(
         output: List[str],
         node_config: Optional[dict] = None,
         node_name: str = "Robots",
-
     ):
         super().__init__(node_name, "node", input, output, 1)
 
         self.llm_model = node_config["llm_model"]
 
-        self.force_scraping = False if node_config is None else node_config.get("force_scraping", False)
+        self.force_scraping = (
+            False if node_config is None else node_config.get("force_scraping", False)
+        )
         self.verbose = (
             True if node_config is None else node_config.get("verbose", False)
         )
@@ -111,14 +110,11 @@ def execute(self, state: dict) -> dict:
             base_url = f"{parsed_url.scheme}://{parsed_url.netloc}"
             loader = AsyncChromiumLoader(f"{base_url}/robots.txt")
             document = loader.load()
-            if "ollama" in self.llm_model["model_name"]:
-                self.llm_model["model_name"] = self.llm_model["model_name"].split("/")[
-                    -1
-                ]
-                model = self.llm_model["model_name"].split("/")[-1]
-
+            if "ollama" in self.llm_model.model_name:
+                self.llm_model.model_name = self.llm_model.model_name.split("/")[-1]
+                model = self.llm_model.model_name.split("/")[-1]
             else:
-                model = self.llm_model["model_name"]
+                model = self.llm_model.model_name
             try:
                 agent = robots_dictionary[model]