Merge pull request #454 from ScrapeGraphAI/removing-rag-node

merge: removed rag node

Author: f-aguzzi

scrapegraphai/graphs/csv_scraper_graph.py

@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerCSVNode
 )
 
@@ -37,14 +36,7 @@ def _create_graph(self):
             input="csv | csv_dir",
             output=["doc"],
         )
-        rag_node = RAGNode(
-            input="user_prompt & doc",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model,
-            }
-        )
+     
         generate_answer_node = GenerateAnswerCSVNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -58,12 +50,10 @@ def _create_graph(self):
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/graphs/json_scraper_graph.py

@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerNode
 )
 
@@ -62,14 +61,7 @@ def _create_graph(self) -> BaseGraph:
             input="json | json_dir",
             output=["doc", "link_urls", "img_urls"],
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+     
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -83,12 +75,10 @@ def _create_graph(self) -> BaseGraph:
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/graphs/markdown_scraper_graph.py

@@ -3,7 +3,7 @@
 from pydantic import BaseModel
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-from ..nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode
+from ..nodes import FetchNode, ParseNode, GenerateAnswerNode
 
 class MDScraperGraph(AbstractGraph):
     """
@@ -63,14 +63,6 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -86,13 +78,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node)
+                (parse_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/graphs/omni_scraper_graph.py

@@ -12,7 +12,6 @@
     FetchNode,
     ParseNode,
     ImageToTextNode,
-    RAGNode,
     GenerateAnswerOmniNode
 )
 
@@ -89,14 +88,7 @@ def _create_graph(self) -> BaseGraph:
                 "max_images": self.max_images
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+      
         generate_answer_omni_node = GenerateAnswerOmniNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc) & img_desc",
             output=["answer"],
@@ -112,14 +104,12 @@ def _create_graph(self) -> BaseGraph:
                 fetch_node,
                 parse_node,
                 image_to_text_node,
-                rag_node,
                 generate_answer_omni_node,
             ],
             edges=[
                 (fetch_node, parse_node),
                 (parse_node, image_to_text_node),
-                (image_to_text_node, rag_node),
-                (rag_node, generate_answer_omni_node)
+                (image_to_text_node, generate_answer_omni_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
@@ -136,4 +126,4 @@ def run(self) -> str:
         inputs = {"user_prompt": self.prompt, self.input_key: self.source}
         self.final_state, self.execution_info = self.graph.execute(inputs)
 
-        return self.final_state.get("answer", "No answer found.")
+        return self.final_state.get("answer", "No answer found.")

scrapegraphai/graphs/pdf_scraper_graph.py

@@ -12,7 +12,6 @@
 from ..nodes import (
     FetchNode,
     ParseNode,
-    RAGNode,
     GenerateAnswerPDFNode
 )
 
@@ -76,14 +75,6 @@ def _create_graph(self) -> BaseGraph:
             }
         )
 
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
         generate_answer_node_pdf = GenerateAnswerPDFNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -98,13 +89,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node_pdf,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node_pdf)
+                (parse_node, generate_answer_node_pdf)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/graphs/smart_scraper_graph.py

@@ -11,7 +11,6 @@
 from ..nodes import (
     FetchNode,
     ParseNode,
-    RAGNode,
     GenerateAnswerNode
 )
 
@@ -78,14 +77,7 @@ def _create_graph(self) -> BaseGraph:
                 "chunk_size": self.model_token
             }
         )
-        rag_node = RAGNode(
-            input="user_prompt & (parsed_doc | doc)",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | parsed_doc | doc)",
             output=["answer"],
@@ -100,13 +92,11 @@ def _create_graph(self) -> BaseGraph:
             nodes=[
                 fetch_node,
                 parse_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
                 (fetch_node, parse_node),
-                (parse_node, rag_node),
-                (rag_node, generate_answer_node)
+                (parse_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/graphs/xml_scraper_graph.py

@@ -10,7 +10,6 @@
 
 from ..nodes import (
     FetchNode,
-    RAGNode,
     GenerateAnswerNode
 )
 
@@ -64,14 +63,7 @@ def _create_graph(self) -> BaseGraph:
             input="xml | xml_dir",
             output=["doc", "link_urls", "img_urls"]
         )
-        rag_node = RAGNode(
-            input="user_prompt & doc",
-            output=["relevant_chunks"],
-            node_config={
-                "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
-            }
-        )
+     
         generate_answer_node = GenerateAnswerNode(
             input="user_prompt & (relevant_chunks | doc)",
             output=["answer"],
@@ -85,12 +77,10 @@ def _create_graph(self) -> BaseGraph:
         return BaseGraph(
             nodes=[
                 fetch_node,
-                rag_node,
                 generate_answer_node,
             ],
             edges=[
-                (fetch_node, rag_node),
-                (rag_node, generate_answer_node)
+                (fetch_node, generate_answer_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -125,7 +125,7 @@ def execute(self, state):
                     template=template_no_chunks_csv_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "format_instructions": format_instructions,
                     },
                 )
@@ -137,7 +137,7 @@ def execute(self, state):
                     template=template_chunks_csv_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "chunk_id": i + 1,
                         "format_instructions": format_instructions,
                     },

scrapegraphai/nodes/generate_answer_node.py

@@ -115,7 +115,7 @@ def execute(self, state: dict) -> dict:
                 prompt = PromptTemplate(
                     template=template_no_chunks_prompt,
                     input_variables=["question"],
-                    partial_variables={"context": chunk.page_content,
+                    partial_variables={"context": chunk,
                                        "format_instructions": format_instructions})
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
@@ -124,7 +124,7 @@ def execute(self, state: dict) -> dict:
                 prompt = PromptTemplate(
                     template=template_chunks_prompt,
                     input_variables=["question"],
-                    partial_variables={"context": chunk.page_content,
+                    partial_variables={"context": chunk,
                                         "chunk_id": i + 1,
                                         "format_instructions": format_instructions})
             # Dynamically name the chains based on their index

scrapegraphai/nodes/generate_answer_omni_node.py

@@ -110,7 +110,7 @@ def execute(self, state: dict) -> dict:
                     template=template_no_chunk_omni_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "format_instructions": format_instructions,
                         "img_desc": imag_desc,
                     },
@@ -123,7 +123,7 @@ def execute(self, state: dict) -> dict:
                     template=template_chunks_omni_prompt,
                     input_variables=["question"],
                     partial_variables={
-                        "context": chunk.page_content,
+                        "context": chunk,
                         "chunk_id": i + 1,
                         "format_instructions": format_instructions,
                     },