Merge branch 'main' into temp-1

VinciGit00 · web-flow · commit d0976ddda931 · 2024-09-25T17:45:45.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
 ## [1.22.0-beta.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.22.0-beta.2...v1.22.0-beta.3) (2024-09-25)
 
 
+
 ### Bug Fixes
 
 * update to pydantic documentation ([76ce257](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/76ce257efb9d9f46c0693472a1fe54b39e4eb1ef))
@@ -15,6 +16,7 @@
 ## [1.22.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.21.2-beta.2...v1.22.0-beta.1) (2024-09-24)
 
 
+
 ### Features
 
 * add info to the dictionary for toghtherai ([3b5ee76](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3b5ee767cbb91cb0ca8e4691195d16c3b57140bb))
diff --git a/pyproject.toml b/pyproject.toml
@@ -3,6 +3,7 @@ name = "scrapegraphai"
 
 version = "1.22.0b3"
 
+
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [
     { name = "Marco Vinciguerra", email = "mvincig11@gmail.com" },
diff --git a/scrapegraphai/graphs/csv_scraper_graph.py b/scrapegraphai/graphs/csv_scraper_graph.py
@@ -1,7 +1,6 @@
 """
 Module for creating the smart scraper
 """
-
 from typing import Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
diff --git a/scrapegraphai/graphs/csv_scraper_multi_graph.py b/scrapegraphai/graphs/csv_scraper_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 CSVScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/json_scraper_multi_graph.py b/scrapegraphai/graphs/json_scraper_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 JSONScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/omni_scraper_graph.py b/scrapegraphai/graphs/omni_scraper_graph.py
@@ -1,7 +1,6 @@
 """
 OmniScraperGraph Module
 """
-
 from typing import Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
diff --git a/scrapegraphai/graphs/omni_search_graph.py b/scrapegraphai/graphs/omni_search_graph.py
@@ -1,7 +1,6 @@
 """ 
 OmniSearchGraph Module
 """
-
 from copy import deepcopy
 from typing import Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/pdf_scraper_graph.py b/scrapegraphai/graphs/pdf_scraper_graph.py
@@ -1,4 +1,3 @@
-
 """
 PDFScraperGraph Module
 """
diff --git a/scrapegraphai/graphs/pdf_scraper_multi_graph.py b/scrapegraphai/graphs/pdf_scraper_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 PdfScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/script_creator_multi_graph.py b/scrapegraphai/graphs/script_creator_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 ScriptCreatorMultiGraph Module
 """
-
 from typing import List, Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
diff --git a/scrapegraphai/graphs/search_graph.py b/scrapegraphai/graphs/search_graph.py
@@ -1,7 +1,6 @@
 """ 
 SearchGraph Module
 """
-
 from copy import deepcopy
 from typing import Optional, List
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/search_link_graph.py b/scrapegraphai/graphs/search_link_graph.py
@@ -6,9 +6,11 @@
 from pydantic import BaseModel
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-from ..nodes import ( FetchNode, ParseNode, SearchLinkNode )
+from ..nodes import (FetchNode,
+                     SearchLinkNode,
+                     SearchLinksWithContext)
 
-class SearchLinkGraph(AbstractGraph): 
+class SearchLinkGraph(AbstractGraph):
     """ 
     SearchLinkGraph is a scraping pipeline that automates the process of 
     extracting information from web pages using a natural language model
@@ -30,13 +32,7 @@ class SearchLinkGraph(AbstractGraph):
         config (dict): Configuration parameters for the graph.
         schema (BaseModel, optional): The schema for the graph output. Defaults to None.
 
-    Example:
-        >>> smart_scraper = SearchLinkGraph(
-        ...     "List me all the attractions in Chioggia.",
-        ...     "https://en.wikipedia.org/wiki/Chioggia",
-        ...     {"llm": {"model": "openai/gpt-3.5-turbo"}}
-        ... )
-        >>> result = smart_scraper.run()
+
     """
 
     def __init__(self, source: str, config: dict, schema: Optional[BaseModel] = None):
@@ -51,45 +47,41 @@ def _create_graph(self) -> BaseGraph:
         Returns:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
-
         fetch_node = FetchNode(
-            input="url| local_dir",
-            output=["doc"],
-            node_config={
-                "llm_model": self.llm_model,
-                "force": self.config.get("force", False),
-                "cut": self.config.get("cut", True),
-                "loader_kwargs": self.config.get("loader_kwargs", {}),
-            }
-        )
-        parse_node = ParseNode(
-            input="doc",
-            output=["parsed_doc"],
-            node_config={
-                "chunk_size": self.model_token,
-                "llm_model": self.llm_model
-            }
-        )
-        search_link_node = SearchLinkNode(
-            input="doc",
-            output=["parsed_doc"],
-            node_config={
-                "llm_model": self.llm_model,
-                "chunk_size": self.model_token,
-                "filter_links": self.config.get("filter_links", None),
-                "filter_config": self.config.get("filter_config", None)
-            }
-        )
+                input="url| local_dir",
+                output=["doc"],
+                node_config={
+                    "force": self.config.get("force", False),
+                    "cut": self.config.get("cut", True),
+                    "loader_kwargs": self.config.get("loader_kwargs", {}),
+                }
+            )
+
+        if self.config.get("llm_style") == (True, None):
+            search_link_node = SearchLinksWithContext(
+                input="doc",
+                output=["parsed_doc"],
+                node_config={
+                    "llm_model": self.llm_model,
+                    "chunk_size": self.model_token,
+                }
+            )
+        else:
+            search_link_node = SearchLinkNode(
+                input="doc",
+                output=["parsed_doc"],
+                node_config={
+                    "chunk_size": self.model_token,
+                }
+            )
 
         return BaseGraph(
             nodes=[
                 fetch_node,
-                parse_node,
                 search_link_node
             ],
             edges=[
-                (fetch_node, parse_node),
-                (parse_node, search_link_node)
+                (fetch_node, search_link_node)
             ],
             entry_point=fetch_node,
             graph_name=self.__class__.__name__
diff --git a/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py b/scrapegraphai/graphs/smart_scraper_multi_concat_graph.py
@@ -1,7 +1,6 @@
 """ 
 SmartScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
@@ -14,7 +13,6 @@
 )
 from ..utils.copy import safe_deepcopy
 
-
 class SmartScraperMultiConcatGraph(AbstractGraph):
     """ 
     SmartScraperMultiGraph is a scraping pipeline that scrapes a 
@@ -43,9 +41,8 @@ class SmartScraperMultiConcatGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], 
+    def __init__(self, prompt: str, source: List[str],
                  config: dict, schema: Optional[BaseModel] = None):
-        
         self.copy_config = safe_deepcopy(config)
 
         self.copy_schema = deepcopy(schema)
diff --git a/scrapegraphai/graphs/smart_scraper_multi_graph.py b/scrapegraphai/graphs/smart_scraper_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 SmartScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/graphs/xml_scraper_graph.py b/scrapegraphai/graphs/xml_scraper_graph.py
@@ -1,7 +1,6 @@
 """
 XMLScraperGraph Module
 """
-
 from typing import Optional
 from pydantic import BaseModel
 from .base_graph import BaseGraph
diff --git a/scrapegraphai/graphs/xml_scraper_multi_graph.py b/scrapegraphai/graphs/xml_scraper_multi_graph.py
@@ -1,7 +1,6 @@
 """ 
 XMLScraperMultiGraph Module
 """
-
 from copy import deepcopy
 from typing import List, Optional
 from pydantic import BaseModel
diff --git a/scrapegraphai/nodes/__init__.py b/scrapegraphai/nodes/__init__.py
@@ -25,4 +25,5 @@
 from .concat_answers_node import ConcatAnswersNode
 from .prompt_refiner_node import PromptRefinerNode
 from .html_analyzer_node import HtmlAnalyzerNode
-from .generate_code_node import GenerateCodeNode
+from .generate_code_node import GenerateCodeNode
+from .search_node_with_context import SearchLinksWithContext
diff --git a/scrapegraphai/nodes/base_node.py b/scrapegraphai/nodes/base_node.py
@@ -1,13 +1,11 @@
 """ 
 BaseNode Module
 """
-
 import re
 from abc import ABC, abstractmethod
 from typing import List, Optional
 from ..utils import get_logger
 
-
 class BaseNode(ABC):
     """
     An abstract base class for nodes in a graph-based workflow, 
diff --git a/scrapegraphai/nodes/concat_answers_node.py b/scrapegraphai/nodes/concat_answers_node.py
@@ -1,7 +1,6 @@
 """
 ConcatAnswersNode Module
 """
-
 from typing import List, Optional
 from ..utils.logging import get_logger
 from .base_node import BaseNode
diff --git a/scrapegraphai/nodes/conditional_node.py b/scrapegraphai/nodes/conditional_node.py
@@ -32,7 +32,6 @@ def __init__(self):
         """
         Initializes an empty ConditionalNode.
         """
-        
         #super().__init__(node_name, "node", input, output, 2, node_config)
         pass
 
diff --git a/scrapegraphai/nodes/fetch_screen_node.py b/scrapegraphai/nodes/fetch_screen_node.py
@@ -16,7 +16,7 @@ def __init__(
         input: str,
         output: List[str],
         node_config: Optional[dict] = None,
-        node_name: str = "FetchScreenNode",
+        node_name: str = "FetchScreen",
     ):
         super().__init__(node_name, "node", input, output, 2, node_config)
         self.url = node_config.get("link")
diff --git a/scrapegraphai/nodes/generate_answer_csv_node.py b/scrapegraphai/nodes/generate_answer_csv_node.py
@@ -1,7 +1,6 @@
 """
 Module for generating the answer node
 """
-
 from typing import List, Optional
 from langchain.prompts import PromptTemplate
 from langchain_core.output_parsers import JsonOutputParser
diff --git a/scrapegraphai/nodes/graph_iterator_node.py b/scrapegraphai/nodes/graph_iterator_node.py
@@ -4,8 +4,8 @@
 import asyncio
 from typing import List, Optional
 from tqdm.asyncio import tqdm
-from .base_node import BaseNode
 from pydantic import BaseModel
+from .base_node import BaseNode
 
 DEFAULT_BATCHSIZE = 16
 
diff --git a/scrapegraphai/nodes/parse_node.py b/scrapegraphai/nodes/parse_node.py
@@ -33,7 +33,7 @@ def __init__(
         input: str,
         output: List[str],
         node_config: Optional[dict] = None,
-        node_name: str = "Parse",
+        node_name: str = "ParseNode",
     ):
         super().__init__(node_name, "node", input, output, 1, node_config)
 
@@ -88,7 +88,7 @@ def execute(self, state: dict) -> dict:
             link_urls, img_urls = self._extract_urls(docs_transformed.page_content, source)
 
             chunk_size = self.chunk_size
-            chunk_size = min(chunk_size - 500, int(chunk_size * 0.9))
+            chunk_size = min(chunk_size - 500, int(chunk_size * 0.75))
 
             if isinstance(docs_transformed, Document):
                 chunks = split_text_into_chunks(text=docs_transformed.page_content,
diff --git a/scrapegraphai/nodes/rag_node.py b/scrapegraphai/nodes/rag_node.py
diff --git a/scrapegraphai/nodes/search_link_node.py b/scrapegraphai/nodes/search_link_node.py
diff --git a/scrapegraphai/nodes/search_node_with_context.py b/scrapegraphai/nodes/search_node_with_context.py

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,3 @@`
`1`		`-`
`2`	`1`	`"""`
`3`	`2`	`PDFScraperGraph Module`
`4`	`3`	`"""`