feat: multiple graph instances

Author: PeriniM

scrapegraphai/nodes/graphs_iterator_node.py renamed to examples/openai/search_graph_multi.py

@@ -6,8 +6,8 @@
 from dotenv import load_dotenv
 from langchain_openai import OpenAIEmbeddings
 from scrapegraphai.models import OpenAI
-from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, SearchInternetNode
+from scrapegraphai.graphs import BaseGraph, SmartScraperGraph
+from scrapegraphai.nodes import SearchInternetNode, GraphIteratorNode, MergeAnswersNode
 load_dotenv()
 
 # ************************************************
@@ -23,6 +23,16 @@
     },
 }
 
+# ************************************************
+# Create a SmartScraperGraph instance
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="",
+    source="",
+    config=graph_config
+)
+
 # ************************************************
 # Define the graph nodes
 # ************************************************
@@ -32,38 +42,24 @@
 
 search_internet_node = SearchInternetNode(
     input="user_prompt",
-    output=["url"],
-    node_config={
-        "llm_model": llm_model
-    }
-)
-fetch_node = FetchNode(
-    input="url | local_dir",
-    output=["doc"],
+    output=["urls"],
     node_config={
-        "verbose": True,
-        "headless": True,
-    }
-)
-parse_node = ParseNode(
-    input="doc",
-    output=["parsed_doc"],
-    node_config={
-        "chunk_size": 4096,
+        "llm_model": llm_model,
         "verbose": True,
     }
 )
-rag_node = RAGNode(
-    input="user_prompt & (parsed_doc | doc)",
-    output=["relevant_chunks"],
+
+graph_iterator_node = GraphIteratorNode(
+    input="user_prompt & urls",
+    output=["results"],
     node_config={
-        "llm_model": llm_model,
-        "embedder_model": embedder,
+        "graph_instance": smart_scraper_graph,
         "verbose": True,
     }
 )
-generate_answer_node = GenerateAnswerNode(
-    input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+
+merge_answers_node = MergeAnswersNode(
+    input="user_prompt & results",
     output=["answer"],
     node_config={
         "llm_model": llm_model,
@@ -78,16 +74,12 @@
 graph = BaseGraph(
     nodes=[
         search_internet_node,
-        fetch_node,
-        parse_node,
-        rag_node,
-        generate_answer_node,
+        graph_iterator_node,
+        merge_answers_node
     ],
     edges=[
-        (search_internet_node, fetch_node),
-        (fetch_node, parse_node),
-        (parse_node, rag_node),
-        (rag_node, generate_answer_node)
+        (search_internet_node, graph_iterator_node),
+        (graph_iterator_node, merge_answers_node)
     ],
     entry_point=search_internet_node
 )

scrapegraphai/nodes/__init__.py

@@ -17,3 +17,5 @@
 from .robots_node import RobotsNode
 from .generate_answer_csv_node import GenerateAnswerCSVNode
 from .generate_answer_pdf_node import GenerateAnswerPDFNode
+from .graph_iterator_node import GraphIteratorNode
+from .merge_answers_node import MergeAnswersNode

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -2,7 +2,7 @@
 Module for generating the answer node
 """
 # Imports from standard library
-from typing import List
+from typing import List, Optional
 from tqdm import tqdm
 
 # Imports from Langchain
@@ -39,7 +39,7 @@ class GenerateAnswerCSVNode(BaseNode):
                         updating the state with the generated answer under the 'answer' key.
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict] = None,
                  node_name: str = "GenerateAnswer"):
         """
         Initializes the GenerateAnswerNodeCsv with a language model client and a node name.

scrapegraphai/nodes/generate_answer_node.py

@@ -3,7 +3,7 @@
 """
 
 # Imports from standard library
-from typing import List
+from typing import List, Optional
 from tqdm import tqdm
 
 # Imports from Langchain
@@ -33,7 +33,7 @@ class GenerateAnswerNode(BaseNode):
         node_name (str): The unique identifier name for the node, defaulting to "GenerateAnswer".
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict]=None,
                  node_name: str = "GenerateAnswer"):
         super().__init__(node_name, "node", input, output, 2, node_config)
 

scrapegraphai/nodes/generate_answer_node_csv.py

@@ -2,7 +2,7 @@
 Module for generating the answer node
 """
 # Imports from standard library
-from typing import List
+from typing import List, Optional
 from tqdm import tqdm
 
 # Imports from Langchain
@@ -39,7 +39,7 @@ class GenerateAnswerCSVNode(BaseNode):
                         updating the state with the generated answer under the 'answer' key.
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict] = None,
                  node_name: str = "GenerateAnswer"):
         """
         Initializes the GenerateAnswerNodeCsv with a language model client and a node name.

scrapegraphai/nodes/generate_answer_pdf_node.py

@@ -2,7 +2,7 @@
 Module for generating the answer node
 """
 # Imports from standard library
-from typing import List
+from typing import List, Optional
 from tqdm import tqdm
 
 # Imports from Langchain
@@ -39,7 +39,7 @@ class GenerateAnswerPDFNode(BaseNode):
                         updating the state with the generated answer under the 'answer' key.
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict] = None,
                  node_name: str = "GenerateAnswer"):
         """
         Initializes the GenerateAnswerNodePDF with a language model client and a node name.

scrapegraphai/nodes/generate_scraper_node.py

@@ -3,7 +3,7 @@
 """
 
 # Imports from standard library
-from typing import List
+from typing import List, Optional
 from tqdm import tqdm
 
 # Imports from Langchain
@@ -36,8 +36,8 @@ class GenerateScraperNode(BaseNode):
 
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
-                 library: str, website: str, node_name: str = "GenerateAnswer"):
+    def __init__(self, input: str, output: List[str], library: str, website: str,
+                 node_config: Optional[dict]=None, node_name: str = "GenerateAnswer"):
         super().__init__(node_name, "node", input, output, 2, node_config)
 
         self.llm_model = node_config["llm_model"]

scrapegraphai/nodes/get_probable_tags_node.py

@@ -2,7 +2,7 @@
 GetProbableTagsNode Module
 """
 
-from typing import List
+from typing import List, Optional
 from langchain.output_parsers import CommaSeparatedListOutputParser
 from langchain.prompts import PromptTemplate
 from .base_node import BaseNode

scrapegraphai/nodes/graph_iterator_node.py

@@ -0,0 +1,83 @@
+"""
+GraphIterator Module
+"""
+
+from typing import List, Optional
+import copy
+from tqdm import tqdm
+from .base_node import BaseNode
+
+
+class GraphIteratorNode(BaseNode):
+    """
+    A node responsible for parsing HTML content from a document. 
+    The parsed content is split into chunks for further processing.
+
+    This node enhances the scraping workflow by allowing for targeted extraction of 
+    content, thereby optimizing the processing of large HTML documents.
+
+    Attributes:
+        verbose (bool): A flag indicating whether to show print statements during execution.
+
+    Args:
+        input (str): Boolean expression defining the input keys needed from the state.
+        output (List[str]): List of output keys to be updated in the state.
+        node_config (dict): Additional configuration for the node.
+        node_name (str): The unique identifier name for the node, defaulting to "Parse".
+    """
+
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict]=None, node_name: str = "GraphIterator"):
+        super().__init__(node_name, "node", input, output, 2, node_config)
+
+        self.verbose = False if node_config is None else node_config.get("verbose", False)
+
+    def execute(self,  state: dict) -> dict:
+        """
+        Executes the node's logic to parse the HTML document content and split it into chunks.
+
+        Args:
+            state (dict): The current state of the graph. The input keys will be used to fetch the
+                            correct data from the state.
+
+        Returns:
+            dict: The updated state with the output key containing the parsed content chunks.
+
+        Raises:
+            KeyError: If the input keys are not found in the state, indicating that the
+                        necessary information for parsing the content is missing.
+        """
+
+        if self.verbose:
+            print(f"--- Executing {self.node_name} Node ---")
+
+        # Interpret input keys based on the provided input expression
+        input_keys = self.get_input_keys(state)
+
+        # Fetching data from the state based on the input keys
+        input_data = [state[key] for key in input_keys]
+
+        user_prompt = input_data[0]
+        urls = input_data[1]
+
+        graph_instance = self.node_config.get("graph_instance", None)
+        if graph_instance is None:
+            raise ValueError("Graph instance is required for graph iteration.")
+        
+        # set the prompt and source for each url
+        graph_instance.prompt = user_prompt
+        graphs_instances = []
+        for url in urls:
+            # make a copy of the graph instance
+            copy_graph_instance = copy.copy(graph_instance)
+            copy_graph_instance.source = url
+            graphs_instances.append(copy_graph_instance)
+
+        # run the graph for each url and use tqdm for progress bar
+        graphs_answers = []
+        for graph in tqdm(graphs_instances, desc="Processing Graph Instances", disable=not self.verbose):
+            result = graph.run()
+            graphs_answers.append(result)
+
+        state.update({self.output[0]: graphs_answers})
+
+        return state

scrapegraphai/nodes/image_to_text_node.py

@@ -2,7 +2,7 @@
 ImageToTextNode Module
 """
 
-from typing import List
+from typing import List, Optional
 from .base_node import BaseNode
 
 
@@ -21,7 +21,7 @@ class ImageToTextNode(BaseNode):
         node_name (str): The unique identifier name for the node, defaulting to "ImageToText".
     """
 
-    def __init__(self, input: str, output: List[str], node_config: dict,
+    def __init__(self, input: str, output: List[str], node_config: Optional[dict]=None,
                  node_name: str = "ImageToText"):
         super().__init__(node_name, "node", input, output, 1, node_config)