refactoring of the code

Co-Authored-By: Matteo Vedovati <[email protected]>

Author: VinciGit00

examples/local_models/smart_scraper_ollama.py

@@ -22,8 +22,13 @@
 # Create the SmartScraperGraph instance and run it
 # ************************************************
 smart_scraper_graph = SmartScraperGraph(
+<<<<<<< Updated upstream
     prompt="Find some information about what does the company do, the name and a contact email.",
     source="https://scrapegraphai.com/",
+=======
+    prompt="List all the projects with their descriptions",
+    source="https://perinim.github.io/projects/",
+>>>>>>> Stashed changes
     config=graph_config
 )
 

scrapegraphai/graphs/abstract_graph.py

@@ -7,20 +7,16 @@
 import uuid
 import warnings
 from pydantic import BaseModel
-
 from langchain_community.chat_models import ErnieBotChat
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 from langchain.chat_models import init_chat_model
-
 from ..helpers import models_tokens
 from ..models import (
     OneApi,
     DeepSeek
 )
 from ..utils.logging import set_verbosity_warning, set_verbosity_info
 
-
-
 class AbstractGraph(ABC):
     """
     Scaffolding class for creating a graph representation and executing it.

scrapegraphai/graphs/base_graph.py

@@ -6,9 +6,7 @@
 from typing import Tuple
 from langchain_community.callbacks import get_openai_callback
 from ..integrations import BurrBridge
-
-# Import telemetry functions
-from ..telemetry import log_graph_execution, log_event
+from ..telemetry import log_graph_execution
 
 class BaseGraph:
     """

scrapegraphai/graphs/csv_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerCSVNode
 )
 
-
 class CSVScraperGraph(AbstractGraph):
     """
     SmartScraper is a comprehensive web scraping tool that automates the process of extracting

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -4,22 +4,19 @@
 
 from copy import copy, deepcopy
 from typing import List, Optional
-
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .csv_scraper_graph import CSVScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class CSVScraperMultiGraph(AbstractGraph):
     """ 
-    CSVScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    CSVScraperMultiGraph is a scraping pipeline that 
+    scrapes a list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
@@ -44,7 +41,8 @@ class CSVScraperMultiGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
+    def __init__(self, prompt: str, source: List[str], 
+                 config: dict, schema: Optional[BaseModel] = None):
 
         self.max_results = config.get("max_results", 3)
 

scrapegraphai/graphs/deep_scraper_graph.py

@@ -4,10 +4,8 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     SearchLinkNode,
@@ -18,7 +16,6 @@
     MergeAnswersNode
 )
 
-
 class DeepScraperGraph(AbstractGraph):
     """
     [WIP]
@@ -87,7 +84,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_chunks"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         generate_answer_node = GenerateAnswerNode(
@@ -104,7 +100,6 @@ def _create_repeated_graph(self) -> BaseGraph:
             output=["relevant_links"],
             node_config={
                 "llm_model": self.llm_model,
-                "embedder_model": self.embedder_model
             }
         )
         graph_iterator_node = GraphIteratorNode(

scrapegraphai/graphs/json_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerNode
 )
 
-
 class JSONScraperGraph(AbstractGraph):
     """
     JSONScraperGraph defines a scraping pipeline for JSON files.

scrapegraphai/graphs/json_scraper_multi_graph.py

@@ -5,20 +5,18 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .json_scraper_graph import JSONScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class JSONScraperMultiGraph(AbstractGraph):
     """ 
-    JSONScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    JSONScraperMultiGraph is a scraping pipeline that scrapes a 
+    list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:

scrapegraphai/graphs/markdown_scraper_multi_graph.py

@@ -5,17 +5,14 @@
 from copy import copy, deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .markdown_scraper_graph import MDScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class MDScraperMultiGraph(AbstractGraph):
     """
     MDScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and 

scrapegraphai/graphs/omni_scraper_graph.py

@@ -4,17 +4,14 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     ParseNode,
     ImageToTextNode,
     GenerateAnswerOmniNode
 )
-
 from ..models import OpenAIImageToText
 
 class OmniScraperGraph(AbstractGraph):