Merge branch 'pre/beta' into refactoring-of-abstract-graph

Author: VinciGit00

CHANGELOG.md

@@ -1,3 +1,22 @@
+## [1.14.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.3...v1.14.0-beta.1) (2024-08-11)
+
+
+### Features
+
+* add refactoring of default temperature ([6c3b37a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6c3b37ab001b80c09ea9ffb56d4c3df338e33a7a))
+
+
+### Bug Fixes
+
+* broken node ([1272273](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/127227349915deeb0dede34aa575ad269ed7cbe3))
+* merge_anwser prompt import ([f17cef9](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/f17cef94bb39349d40cc520d93b51ac4e629db32))
+
+
+### CI
+
+* **release:** 1.13.0-beta.8 [skip ci] ([b470d97](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/b470d974cf3fdb3a75ead46fceb8c21525e2e616))
+* **release:** 1.13.0-beta.9 [skip ci] ([d4c1a1c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d4c1a1c58a54740ff50aa87b1d1d3500b61ea088))
+
 ## [1.13.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.2...v1.13.3) (2024-08-10)
 
 
@@ -25,6 +44,8 @@
 * conditional node ([ce00345](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/ce003454953e5785d4746223c252de38cd5d07ea))
 
 ## [1.13.0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.12.2...v1.13.0) (2024-08-09)
+## [1.13.0-beta.9](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.8...v1.13.0-beta.9) (2024-08-10)
+
 
 
 ### Features
@@ -65,6 +86,15 @@
 * **release:** 1.13.0-beta.5 [skip ci] ([2eba73b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2eba73b784ee443260117e98ab7c943934b3018d)), closes [#513](https://github.com/ScrapeGraphAI/Scrapegraph-ai/issues/513)
 * **release:** 1.13.0-beta.6 [skip ci] ([e75b574](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/e75b574b67040e127599da9ee1b0eee13d234cb9))
 * **release:** 1.13.0-beta.7 [skip ci] ([6e56925](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6e56925355c424edae290c70fd98646ab5f420ee))
+* add refactoring of default temperature ([6c3b37a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/6c3b37ab001b80c09ea9ffb56d4c3df338e33a7a))
+
+## [1.13.0-beta.8](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.7...v1.13.0-beta.8) (2024-08-09)
+
+
+### Bug Fixes
+
+* broken node ([1272273](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/127227349915deeb0dede34aa575ad269ed7cbe3))
+
 
 ## [1.13.0-beta.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.13.0-beta.6...v1.13.0-beta.7) (2024-08-09)
 

examples/local_models/script_generator_ollama.py

@@ -9,16 +9,11 @@
 
 graph_config = {
     "llm": {
-        "model": "ollama/mistral",
-        "temperature": 0,
+        "model": "ollama/llama3.1",
+        "temperature": 0.5,
         # "model_tokens": 2000, # set context length arbitrarily,
         "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
     },
-    "embeddings": {
-        "model": "ollama/nomic-embed-text",
-        "temperature": 0,
-        "base_url": "http://localhost:11434",  # set ollama URL arbitrarily
-    },
     "library": "beautifoulsoup",
     "verbose": True,
 }

examples/local_models/smart_scraper_ollama.py

@@ -14,7 +14,6 @@
         "format": "json",  # Ollama needs the format to be specified explicitly
         # "base_url": "http://localhost:11434", # set ollama URL arbitrarily
     },
-  
     "verbose": True,
     "headless": False
 }

pyproject.toml

@@ -2,7 +2,7 @@
 name = "scrapegraphai"
 
 
-version = "1.13.3"
+version = "1.14.0b1"
 
 
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
@@ -23,6 +23,8 @@ dependencies = [
     "langchain-groq>=0.1.3",
     "langchain-aws>=0.1.3",
     "langchain-anthropic>=0.1.11",
+    "langchain-mistralai>=0.1.12",
+    "langchain-huggingface>=0.0.3",
     "langchain-nvidia-ai-endpoints>=0.1.6",
     "html2text>=2024.2.26",
     "faiss-cpu>=1.8.0",
@@ -38,11 +40,7 @@ dependencies = [
     "google>=3.0.0",
     "undetected-playwright>=0.3.0",
     "semchunk>=1.0.1",
-    "langchain-fireworks>=0.1.3",
-    "langchain-community>=0.2.9",
-    "langchain-huggingface>=0.0.3",
     "browserbase>=0.3.0",
-    "langchain-mistralai>=0.1.12",
 ]
 
 license = "MIT"

requirements-dev.lock

@@ -6,6 +6,8 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiofiles==24.1.0
@@ -110,6 +112,7 @@ filelock==3.15.4
     # via huggingface-hub
     # via torch
     # via transformers
+    # via triton
 fireworks-ai==0.14.0
     # via langchain-fireworks
 fonttools==4.53.1
@@ -185,6 +188,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -358,6 +362,34 @@ numpy==1.26.4
     # via shapely
     # via streamlit
     # via transformers
+nvidia-cublas-cu12==12.1.3.1
+    # via nvidia-cudnn-cu12
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==8.9.2.26
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-nccl-cu12==2.19.3
+    # via torch
+nvidia-nvjitlink-cu12==12.6.20
+    # via nvidia-cusolver-cu12
+    # via nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
 openai==1.37.0
     # via burr
     # via langchain-fireworks
@@ -599,6 +631,8 @@ tqdm==4.66.4
 transformers==4.43.3
     # via langchain-huggingface
     # via sentence-transformers
+triton==2.2.0
+    # via torch
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.12.2
@@ -642,6 +676,8 @@ uvicorn==0.30.3
     # via fastapi
 uvloop==0.19.0
     # via uvicorn
+watchdog==4.0.2
+    # via streamlit
 watchfiles==0.22.0
     # via uvicorn
 websockets==12.0

requirements.lock

@@ -6,6 +6,8 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
+#   generate-hashes: false
+#   universal: false
 
 -e file:.
 aiohttp==3.9.5
@@ -67,6 +69,7 @@ filelock==3.15.4
     # via huggingface-hub
     # via torch
     # via transformers
+    # via triton
 fireworks-ai==0.14.0
     # via langchain-fireworks
 free-proxy==1.1.1
@@ -133,6 +136,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -263,6 +267,34 @@ numpy==1.26.4
     # via sentence-transformers
     # via shapely
     # via transformers
+nvidia-cublas-cu12==12.1.3.1
+    # via nvidia-cudnn-cu12
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-cuda-cupti-cu12==12.1.105
+    # via torch
+nvidia-cuda-nvrtc-cu12==12.1.105
+    # via torch
+nvidia-cuda-runtime-cu12==12.1.105
+    # via torch
+nvidia-cudnn-cu12==8.9.2.26
+    # via torch
+nvidia-cufft-cu12==11.0.2.54
+    # via torch
+nvidia-curand-cu12==10.3.2.106
+    # via torch
+nvidia-cusolver-cu12==11.4.5.107
+    # via torch
+nvidia-cusparse-cu12==12.1.0.106
+    # via nvidia-cusolver-cu12
+    # via torch
+nvidia-nccl-cu12==2.19.3
+    # via torch
+nvidia-nvjitlink-cu12==12.6.20
+    # via nvidia-cusolver-cu12
+    # via nvidia-cusparse-cu12
+nvidia-nvtx-cu12==12.1.105
+    # via torch
 openai==1.37.0
     # via langchain-fireworks
     # via langchain-openai
@@ -414,6 +446,8 @@ tqdm==4.66.4
 transformers==4.43.3
     # via langchain-huggingface
     # via sentence-transformers
+triton==2.2.0
+    # via torch
 typing-extensions==4.12.2
     # via anthropic
     # via anyio

scrapegraphai/graphs/abstract_graph.py

@@ -7,20 +7,16 @@
 import uuid
 import warnings
 from pydantic import BaseModel
-
 from langchain_community.chat_models import ErnieBotChat
 from langchain_nvidia_ai_endpoints import ChatNVIDIA
 from langchain.chat_models import init_chat_model
-
 from ..helpers import models_tokens
 from ..models import (
     OneApi,
     DeepSeek
 )
 from ..utils.logging import set_verbosity_warning, set_verbosity_info
 
-
-
 class AbstractGraph(ABC):
     """
     Scaffolding class for creating a graph representation and executing it.
@@ -53,6 +49,9 @@ class AbstractGraph(ABC):
     def __init__(self, prompt: str, config: dict,
                  source: Optional[str] = None, schema: Optional[BaseModel] = None):
 
+        if config.get("llm").get("temperature") is None:
+            config["llm"]["temperature"] = 0
+
         self.prompt = prompt
         self.source = source
         self.config = config
@@ -163,7 +162,6 @@ def handle_model(model_name, provider, token_key, default_token=8192):
 
         elif llm_params["model"].startswith("vertexai"):
             return handle_model(llm_params["model"], "google_vertexai", llm_params["model"])
-
         elif "gpt-" in llm_params["model"]:
             return handle_model(llm_params["model"], "openai", llm_params["model"])
 
@@ -197,6 +195,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
             return ErnieBotChat(llm_params)
 
         elif "oneapi" in llm_params["model"]:
+
             # take the model after the last dash
             llm_params["model"] = llm_params["model"].split("/")[-1]
             try:
@@ -206,6 +205,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
             return OneApi(llm_params)
 
         elif "nvidia" in llm_params["model"]:
+
             try:
                 self.model_token = models_tokens["nvidia"][llm_params["model"].split("/")[-1]]
                 llm_params["model"] = "/".join(llm_params["model"].split("/")[1:])

scrapegraphai/graphs/base_graph.py

@@ -6,9 +6,7 @@
 from typing import Tuple
 from langchain_community.callbacks import get_openai_callback
 from ..integrations import BurrBridge
-
-# Import telemetry functions
-from ..telemetry import log_graph_execution, log_event
+from ..telemetry import log_graph_execution
 
 class BaseGraph:
     """

scrapegraphai/graphs/csv_scraper_graph.py

@@ -4,16 +4,13 @@
 
 from typing import Optional
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
-
 from ..nodes import (
     FetchNode,
     GenerateAnswerCSVNode
 )
 
-
 class CSVScraperGraph(AbstractGraph):
     """
     SmartScraper is a comprehensive web scraping tool that automates the process of extracting

scrapegraphai/graphs/csv_scraper_multi_graph.py

@@ -4,22 +4,19 @@
 
 from copy import copy, deepcopy
 from typing import List, Optional
-
 from pydantic import BaseModel
-
 from .base_graph import BaseGraph
 from .abstract_graph import AbstractGraph
 from .csv_scraper_graph import CSVScraperGraph
-
 from ..nodes import (
     GraphIteratorNode,
     MergeAnswersNode
 )
 
-
 class CSVScraperMultiGraph(AbstractGraph):
     """ 
-    CSVScraperMultiGraph is a scraping pipeline that scrapes a list of URLs and generates answers to a given prompt.
+    CSVScraperMultiGraph is a scraping pipeline that 
+    scrapes a list of URLs and generates answers to a given prompt.
     It only requires a user prompt and a list of URLs.
 
     Attributes:
@@ -44,7 +41,8 @@ class CSVScraperMultiGraph(AbstractGraph):
         >>> result = search_graph.run()
     """
 
-    def __init__(self, prompt: str, source: List[str], config: dict, schema: Optional[BaseModel] = None):
+    def __init__(self, prompt: str, source: List[str], 
+                 config: dict, schema: Optional[BaseModel] = None):
 
         self.max_results = config.get("max_results", 3)