Merge pull request #254 from mayurdb/nDeep

feat: n-level deep search

Author: VinciGit00

examples/openai/deep_scraper_openai.py

@@ -22,6 +22,7 @@
         "model": "gpt-3.5-turbo",
     },
     "verbose": True,
+    "max_depth": 1
 }
 
 # ************************************************
@@ -31,7 +32,7 @@
 deep_scraper_graph = DeepScraperGraph(
     prompt="List me all the contacts",
     # also accepts a string with the already downloaded HTML code
-    source="https://davittoriogift.com/en/home/a",
+    source="https://www.uber.com/us/en/careers/list/?query=",
     config=graph_config
 )
 

scrapegraphai/graphs/base_graph.py

@@ -132,7 +132,11 @@ def execute(self, initial_state: dict) -> Tuple[dict, list]:
                 cb_total["successful_requests"] += cb["successful_requests"]
                 cb_total["total_cost_USD"] += cb["total_cost_USD"]
 
-                if current_node in self.edges:
+                # Do not execute the graph from this point on if previous node gave a signal
+                if 'skip_branch' in result:
+                    print(f"---- Not executing sub-graph since {current_node.node_name} \
+                    raised a stop signal ---")
+                elif current_node in self.edges:
                     current_node_connections = self.edges[current_node]
                     if current_node.node_type == 'conditional_node':
                         # Assert that there are exactly two out edges from the conditional node

scrapegraphai/graphs/deep_scraper_graph.py

@@ -8,7 +8,9 @@
     SearchLinkNode,
     ParseNode,
     RAGNode,
-    GenerateAnswerNode
+    GenerateAnswerNode,
+    GraphIteratorNode,
+    MergeAnswersNode
 )
 from .abstract_graph import AbstractGraph
 
@@ -18,12 +20,11 @@ class DeepScraperGraph(AbstractGraph):
     [WIP]
 
     DeepScraper is a scraping pipeline that automates the process of 
-    extracting information from web pages
-    using a natural language model to interpret and answer prompts.
-
-    Unlike SmartScraper, DeepScraper can navigate to the links within the input webpage,
-    to fuflfil the task within the prompt.
+    extracting information from web pages using a natural language model 
+    to interpret and answer prompts.
 
+    Unlike SmartScraper, DeepScraper can navigate to the links within,
+    the input webpage to fuflfil the task within the prompt.
     
     Attributes:
         prompt (str): The prompt for the graph.
@@ -50,12 +51,13 @@ class DeepScraperGraph(AbstractGraph):
 
     def __init__(self, prompt: str, source: str, config: dict):
         super().__init__(prompt, config, source)
-
         self.input_key = "url" if source.startswith("http") else "local_dir"
 
-    def _create_graph(self) -> BaseGraph:
+    def _create_repeated_graph(self) -> BaseGraph:
         """
-        Creates the graph of nodes representing the workflow for web scraping.
+        Creates the graph that can be repeatedly executed to conduct search on
+        hyperlinks within the webpage.
+
         Returns:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
@@ -78,6 +80,13 @@ def _create_graph(self) -> BaseGraph:
                 "embedder_model": self.embedder_model
             }
         )
+        generate_answer_node = GenerateAnswerNode(
+            input="user_prompt & (relevant_chunks | parsed_doc | doc)",
+            output=["answer"],
+            node_config={
+                "llm_model": self.llm_model
+            }
+        )
         search_node = SearchLinkNode(
             input="user_prompt & relevant_chunks",
             output=["relevant_links"],
@@ -86,23 +95,60 @@ def _create_graph(self) -> BaseGraph:
                 "embedder_model": self.embedder_model
             }
         )
+        graph_iterator_node = GraphIteratorNode(
+            input="user_prompt & relevant_links",
+            output=["results"],
+            node_config={
+                "graph_instance": None,
+                "batchsize": 1
+            }
+        )
+        merge_answers_node = MergeAnswersNode(
+            input="user_prompt & results",
+            output=["answer"],
+            node_config={
+                "llm_model": self.llm_model,
+            }
+        )
 
         return BaseGraph(
             nodes=[
                 fetch_node,
                 parse_node,
                 rag_node,
-                search_node
+                generate_answer_node,
+                search_node,
+                graph_iterator_node,
+                merge_answers_node
             ],
             edges=[
                 (fetch_node, parse_node),
                 (parse_node, rag_node),
-                (rag_node, search_node)
-
+                (rag_node, generate_answer_node),
+                (rag_node, search_node),
+                (search_node, graph_iterator_node),
+                (graph_iterator_node, merge_answers_node)
             ],
             entry_point=fetch_node
         )
 
+
+
+    def _create_graph(self) -> BaseGraph:
+        """
+        Creates the graph of nodes representing the workflow for web scraping
+        n-levels deep.
+
+        Returns:
+            BaseGraph: A graph instance representing the web scraping workflow.
+        """
+
+        base_graph = self._create_repeated_graph()
+        graph_iterator_node = list(filter(lambda x: x.node_name == "GraphIterator", base_graph.nodes))[0]
+        # Graph iterator will repeat the same graph for multiple hyperlinks found within input webpage
+        graph_iterator_node.node_config["graph_instance"] = self
+        return base_graph
+
     def run(self) -> str:
         """
         Executes the scraping process and returns the answer to the prompt.

scrapegraphai/nodes/graph_iterator_node.py

@@ -104,6 +104,19 @@ async def _async_execute(self, state: dict, batchsize: int) -> dict:
         if graph_instance is None:
             raise ValueError("graph instance is required for concurrent execution")
 
+        # Assign depth level to the graph
+        if "graph_depth" in graph_instance.config:
+            graph_instance.config["graph_depth"] += 1
+        else:
+            graph_instance.config["graph_depth"] = 1
+
+        # Check if max depth is reached
+        if "max_depth" in graph_instance.config and \
+        graph_instance.config["graph_depth"] > graph_instance.config["max_depth"]:
+            print("Max search depth is reached. Terminating search")
+            state.update({"skip_branch": "True"})
+            return state
+
         # sets the prompt for the graph instance
         graph_instance.prompt = user_prompt