Merge pull request #466 from ScrapeGraphAI/refactoring_convert_to_md

refactoring of generate answer node

Author: PeriniM

examples/local_models/smart_scraper_ollama.py

@@ -29,7 +29,7 @@
 
 smart_scraper_graph = SmartScraperGraph(
     prompt="List me all the titles",
-    source="https://sport.sky.it/nba?gr=www",
+    source="https://perinim.github.io/projects",
     config=graph_config
 )
 

scrapegraphai/nodes/base_node.py

@@ -88,7 +88,6 @@ def update_config(self, params: dict, overwrite: bool = False):
             param (dict): The dictionary to update node_config with.
             overwrite (bool): Flag indicating if the values of node_config should be overwritten if their value is not None.
         """
-        
         for key, val in params.items():
             if hasattr(self, key) and not overwrite:
                 continue

scrapegraphai/nodes/generate_answer_csv_node.py

@@ -132,8 +132,9 @@ def execute(self, state):
 
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
-            else:
-                prompt = PromptTemplate(
+                break
+
+            prompt = PromptTemplate(
                     template=template_chunks_csv_prompt,
                     input_variables=["question"],
                     partial_variables={

scrapegraphai/nodes/generate_answer_node.py

@@ -119,8 +119,8 @@ def execute(self, state: dict) -> dict:
                                        "format_instructions": format_instructions})
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
+                break
 
-            else:
                 prompt = PromptTemplate(
                     template=template_chunks_prompt,
                     input_variables=["question"],

scrapegraphai/nodes/generate_answer_omni_node.py

@@ -118,8 +118,9 @@ def execute(self, state: dict) -> dict:
 
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
-            else:
-                prompt = PromptTemplate(
+                break
+
+            prompt = PromptTemplate(
                     template=template_chunks_omni_prompt,
                     input_variables=["question"],
                     partial_variables={

scrapegraphai/nodes/generate_answer_pdf_node.py

@@ -131,8 +131,8 @@ def execute(self, state):
                 chain =  prompt | self.llm_model | output_parser
                 answer = chain.invoke({"question": user_prompt})
 
-            else:
-                prompt = PromptTemplate(
+                break
+            prompt = PromptTemplate(
                     template=template_chunks_pdf_prompt,
                     input_variables=["question"],
                     partial_variables={

scrapegraphai/nodes/parse_node.py

@@ -75,23 +75,23 @@ def execute(self, state: dict) -> dict:
 
             chunks = chunk(text=docs_transformed.page_content,
                             chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                            token_counter=lambda x: len(x.split()),
+                            token_counter=lambda x: len(x),
                             memoize=False)
         else:
             docs_transformed = docs_transformed[0]
 
             if type(docs_transformed) == Document:
                 chunks = chunk(text=docs_transformed.page_content,
                             chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                            token_counter=lambda x: len(x.split()),
+                            token_counter=lambda x: len(x),
                             memoize=False)
             else:
-
+                
                 chunks = chunk(text=docs_transformed,
                                 chunk_size= self.node_config.get("chunk_size", 4096)-250,
-                                token_counter=lambda x: len(x.split()),
+                                token_counter=lambda x: len(x),
                                 memoize=False)
-
+                          
         state.update({self.output[0]: chunks})
 
-        return state
+        return state