refactoring of the code

Author: VinciGit00

scrapegraphai/graphs/abstract_graph.py

@@ -128,7 +128,8 @@ def _create_llm(self, llm_config: dict) -> object:
             if requests_per_second is not None:
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore")
-                    llm_params["rate_limiter"] = InMemoryRateLimiter(requests_per_second=requests_per_second)
+                    llm_params["rate_limiter"] = InMemoryRateLimiter(
+                                                                    requests_per_second=requests_per_second)
             if max_retries is not None:
                 llm_params["max_retries"] = max_retries
 

scrapegraphai/graphs/base_graph.py

@@ -59,7 +59,7 @@ def __init__(self, nodes: list, edges: list, entry_point: str,
             # raise a warning if the entry point is not the first node in the list
             warnings.warn(
                 "Careful! The entry point node is different from the first node in the graph.")
-            
+
         self._set_conditional_node_edges()
 
         # Burr configuration
@@ -89,11 +89,9 @@ def _set_conditional_node_edges(self):
         """
         for node in self.nodes:
             if node.node_type == 'conditional_node':
-                # Find outgoing edges from this ConditionalNode
                 outgoing_edges = [(from_node, to_node) for from_node, to_node in self.raw_edges if from_node.node_name == node.node_name]
                 if len(outgoing_edges) != 2:
                     raise ValueError(f"ConditionalNode '{node.node_name}' must have exactly two outgoing edges.")
-                # Assign true_node_name and false_node_name
                 node.true_node_name = outgoing_edges[0][1].node_name
                 node.false_node_name = outgoing_edges[1][1].node_name
 

scrapegraphai/graphs/code_generator_graph.py

@@ -99,6 +99,7 @@ def _create_graph(self) -> BaseGraph:
                 "schema": self.schema,
             }
         )
+
         prompt_refier_node = PromptRefinerNode(
             input="user_prompt",
             output=["refined_prompt"],
@@ -108,6 +109,7 @@ def _create_graph(self) -> BaseGraph:
                 "schema": self.schema
             }
         )
+
         html_analyzer_node = HtmlAnalyzerNode(
             input="refined_prompt & original_html",
             output=["html_info", "reduced_html"],
@@ -118,6 +120,7 @@ def _create_graph(self) -> BaseGraph:
                 "reduction": self.config.get("reduction", 0)
             }
         )
+
         generate_code_node = GenerateCodeNode(
             input="user_prompt & refined_prompt & html_info & reduced_html & answer",
             output=["generated_code"],

scrapegraphai/graphs/csv_scraper_graph.py

@@ -59,6 +59,7 @@ def _create_graph(self):
         """
         Creates the graph of nodes representing the workflow for web scraping.
         """
+    
         fetch_node = FetchNode(
             input="csv | csv_dir",
             output=["doc"],
@@ -90,6 +91,7 @@ def run(self) -> str:
         """
         Executes the web scraping process and returns the answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -95,6 +95,7 @@ def run(self) -> str:
         Returns:
             str: The answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt, "jsons": self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 

scrapegraphai/graphs/document_scraper_multi_graph.py

@@ -94,6 +94,7 @@ def run(self) -> str:
         Returns:
             str: The answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt, "xmls": self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -96,6 +96,7 @@ def run(self) -> str:
         Returns:
             str: The answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt, "jsons": self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 

scrapegraphai/graphs/omni_scraper_graph.py

@@ -62,13 +62,15 @@ def _create_graph(self) -> BaseGraph:
         Returns:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
+
         fetch_node = FetchNode(
             input="url | local_dir",
             output=["doc"],
             node_config={
                 "loader_kwargs": self.config.get("loader_kwargs", {}),
             }
         )
+
         parse_node = ParseNode(
             input="doc & (url | local_dir)",
             output=["parsed_doc", "link_urls", "img_urls"],
@@ -78,6 +80,7 @@ def _create_graph(self) -> BaseGraph:
                 "llm_model": self.llm_model
             }
         )
+
         image_to_text_node = ImageToTextNode(
             input="img_urls",
             output=["img_desc"],

scrapegraphai/graphs/omni_search_graph.py

@@ -59,13 +59,6 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping and searching workflow.
         """
 
-        # omni_scraper_instance = OmniScraperGraph(
-        #     prompt="",
-        #     source="",
-        #     config=self.copy_config,
-        #     schema=self.copy_schema
-        # )
-
         search_internet_node = SearchInternetNode(
             input="user_prompt",
             output=["urls"],
@@ -115,6 +108,7 @@ def run(self) -> str:
         Returns:
             str: The answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 

scrapegraphai/graphs/script_creator_multi_graph.py

@@ -93,6 +93,7 @@ def run(self) -> str:
         Returns:
             str: The answer to the prompt.
         """
+
         inputs = {"user_prompt": self.prompt, "urls": self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
         return self.final_state.get("merged_script", "Failed to generate the script.")