Merge branch 'pre/beta' into fix/fetch-node-proxybroker

Author: PeriniM

CHANGELOG.md

@@ -1,3 +1,23 @@
+## [0.11.0-beta.4](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.11.0-beta.3...v0.11.0-beta.4) (2024-05-12)
+
+
+### Features
+
+* add new prompt info ([e2350ed](https://github.com/VinciGit00/Scrapegraph-ai/commit/e2350eda6249d8e121344d12c92645a3887a5b76))
+
+## [0.11.0-beta.3](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.11.0-beta.2...v0.11.0-beta.3) (2024-05-12)
+
+
+### Features
+
+* add support for deepseek-chat ([156b67b](https://github.com/VinciGit00/Scrapegraph-ai/commit/156b67b91e1798f67082123e2c0087d358a32d4d)), closes [#222](https://github.com/VinciGit00/Scrapegraph-ai/issues/222)
+
+
+### Docs
+
+* add diagram showing general structure/flow of the library ([13ae918](https://github.com/VinciGit00/Scrapegraph-ai/commit/13ae9180ac5e7ef11dad1a210cf8790e797397dd))
+* update overview diagram with more models ([b441b30](https://github.com/VinciGit00/Scrapegraph-ai/commit/b441b30a5c60dda105964f69bd4cef06825f5c74))
+
 ## [0.11.0-beta.2](https://github.com/VinciGit00/Scrapegraph-ai/compare/v0.11.0-beta.1...v0.11.0-beta.2) (2024-05-10)
 
 

docs/assets/project_overview_diagram.fig

@@ -22,4 +22,14 @@ ScrapegraphAI, leveraging the power of LLMs, adapts to changes in website struct
 This flexibility ensures that scrapers remain functional even when website layouts change.
 
 We support many Large Language Models (LLMs) including GPT, Gemini, Groq, Azure, Hugging Face etc.
-as well as local models which can run on your machine using Ollama.
+as well as local models which can run on your machine using Ollama.
+
+Diagram
+=======
+With ScrapegraphAI you first construct a pipeline of steps you want to execute by combining nodes into a graph.
+Executing the graph takes care of all the steps that are often part of scraping: fetching, parsing etc...
+Finally the scraped and processed data gets fed to an LLM which generates a response.
+
+.. image:: ../../assets/project_overview_diagram.png
+   :align: center
+   :alt: ScrapegraphAI Overview

docs/assets/project_overview_diagram.png

@@ -0,0 +1,47 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+deepseek_key = os.getenv("DEEPSEEK_APIKEY")
+
+graph_config = {
+    "llm": {
+        "model": "deepseek-chat",
+        "openai_api_key": deepseek_key,
+        "openai_api_base": 'https://api.deepseek.com/v1',
+    },
+    "verbose": True,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="List me all the projects with their description.",
+    # also accepts a string with the already downloaded HTML code
+    source="https://perinim.github.io/projects/",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

docs/source/introduction/overview.rst

@@ -0,0 +1,47 @@
+""" 
+Basic example of scraping pipeline using SmartScraper
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import DeepScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+load_dotenv()
+
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-4",
+    },
+    "verbose": True,
+}
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+deep_scraper_graph = DeepScraperGraph(
+    prompt="List me all the job titles and detailed job description.",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.google.com/about/careers/applications/jobs/results/?location=Bangalore%20India",
+    config=graph_config
+)
+
+result = deep_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = deep_scraper_graph.get_execution_info()
+print(deep_scraper_graph.get_state("relevant_links"))
+print(prettify_exec_info(graph_exec_info))

examples/deepseek/smart_scarper_deepseek.py

@@ -27,20 +27,20 @@
 # Create the ScriptCreatorGraph instance and run it
 # ************************************************
 
-smart_scraper_graph = ScriptCreatorGraph(
-    prompt="List me all the news with their description.",
+script_creator_graph = ScriptCreatorGraph(
+    prompt="List me all the projects with their description.",
     # also accepts a string with the already downloaded HTML code
     source="https://perinim.github.io/projects",
     config=graph_config
 )
 
-result = smart_scraper_graph.run()
+result = script_creator_graph.run()
 print(result)
 
 # ************************************************
 # Get graph execution info
 # ************************************************
 
-graph_exec_info = smart_scraper_graph.get_execution_info()
+graph_exec_info = script_creator_graph.get_execution_info()
 print(prettify_exec_info(graph_exec_info))
 

examples/openai/deep_scraper_openai.py

@@ -1,7 +1,7 @@
 [tool.poetry]
 name = "scrapegraphai"
 
-version = "0.11.0b2"
+version = "0.11.0b4"
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
 authors = [

examples/openai/script_generator_openai.py

@@ -5,6 +5,7 @@
 from .abstract_graph import AbstractGraph
 from .base_graph import BaseGraph
 from .smart_scraper_graph import SmartScraperGraph
+from .deep_scraper_graph import DeepScraperGraph
 from .speech_graph import SpeechGraph
 from .search_graph import SearchGraph
 from .script_creator_graph import ScriptCreatorGraph

pyproject.toml

@@ -7,7 +7,7 @@
 from langchain_community.embeddings import HuggingFaceHubEmbeddings, OllamaEmbeddings, BedrockEmbeddings
 from langchain_google_genai import GoogleGenerativeAIEmbeddings
 from ..helpers import models_tokens
-from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI, Anthropic, Claude
+from ..models import AzureOpenAI, Bedrock, Gemini, Groq, HuggingFace, Ollama, OpenAI, Anthropic, Claude, DeepSeek
 
 
 class AbstractGraph(ABC):
@@ -203,6 +203,12 @@ def _create_llm(self, llm_config: dict, chat=False) -> object:
         elif "claude-3-" in llm_params["model"]:
             self.model_token = models_tokens["claude"]["claude3"]
             return Anthropic(llm_params)
+        elif "deepseek" in llm_params["model"]:
+            try:
+                self.model_token = models_tokens["deepseek"][llm_params["model"]]
+            except KeyError as exc:
+                raise KeyError("Model not supported") from exc
+            return DeepSeek(llm_params)
         else:
             raise ValueError(
                 "Model provided by the configuration not supported")