Merge branch 'main' into temp

Author: VinciGit00

CHANGELOG.md

@@ -18,17 +18,37 @@
 ### Features
 
 * add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb))
+* add gpt4o omni ([431edb7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/431edb7bb2504f4c1335c3ae3ce2f91867fa7222))
+* add searchngx integration ([5c92186](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c9218608140bf694fbfd96aa90276bc438bb475))
+* refactoring_to_md function ([602dd00](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/602dd00209ee1d72a1223fc4793759450921fcf9))
+
+
 
 
 ### chore
 
 * **pyproject:** upgrade dependencies ([0425124](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0425124c570f765b98fcf67ba6649f4f9fe76b15))
 
 
+
 ### Docs
 
 * **gpt-4o-mini:** added new gpt, fixed chromium lazy loading, ([99dc849](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/99dc8497d85289759286a973e4aecc3f924d3ada))
 
+
+
+### CI
+
+* **release:** 1.10.0-beta.1 [skip ci] ([8f619de](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/8f619de23540216934b53bcf3426702e56c48f31))
+* **release:** 1.10.0-beta.2 [skip ci] ([aa7d4f0](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/aa7d4f0ebfc2623a51ce1e4887ff26c9906b0a95))
+* **release:** 1.10.0-beta.3 [skip ci] ([bf0a2f3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bf0a2f386f38cbe81d1e5ea3e05357f8ecabcab2))
+* **release:** 1.10.0-beta.4 [skip ci] ([a91807a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/a91807a20cc07b15feb1ddd5cf7a1c323ff32b46))
+* **release:** 1.10.0-beta.5 [skip ci] ([0d5f925](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0d5f9259d8fb148de7c95cf6f67f9562c5d2c880))
+* **release:** 1.9.0-beta.3 [skip ci] ([d3e63d9](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/d3e63d91be79f74e8a3fdb00e692d546c24cead5))
+* **release:** 1.9.0-beta.4 [skip ci] ([2fa04b5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/2fa04b58159abf7af890ebc0768fe23d51bf177f))
+* **release:** 1.9.0-beta.5 [skip ci] ([bb62439](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/bb624399cfc3924825892dd48697fc298ad3b002))
+* **release:** 1.9.0-beta.6 [skip ci] ([54a69de](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/54a69de69e8077e02fd5584783ca62cc2e0ec5bb))
+
 ## [1.10.0-beta.5](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.4...v1.10.0-beta.5) (2024-07-20)
 
 

examples/local_models/smart_scraper_ollama.py

@@ -28,8 +28,8 @@
 # ************************************************
 
 smart_scraper_graph = SmartScraperGraph(
-    prompt="List me all the titles",
-    source="https://perinim.github.io/projects",
+    prompt="List me all the titles of the website",
+    source="https://sport.sky.it/nba?gr=www",
     config=graph_config
 )
 

requirements-dev.lock

@@ -51,6 +51,7 @@ blinker==1.8.2
 boto3==1.34.146
     # via langchain-aws
 botocore==1.34.146
+
     # via boto3
     # via s3transfer
 burr==0.22.1
@@ -179,6 +180,7 @@ greenlet==3.0.3
     # via playwright
     # via sqlalchemy
 groq==0.9.0
+
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
     # via google-cloud-resource-manager
@@ -253,6 +255,7 @@ langchain==0.2.10
 langchain-anthropic==0.1.20
     # via scrapegraphai
 langchain-aws==0.1.12
+
     # via scrapegraphai
 langchain-core==0.2.22
     # via langchain
@@ -268,15 +271,17 @@ langchain-core==0.2.22
 langchain-fireworks==0.1.5
     # via scrapegraphai
 langchain-google-genai==1.0.8
+
     # via scrapegraphai
 langchain-google-vertexai==1.0.7
     # via scrapegraphai
 langchain-groq==0.1.6
     # via scrapegraphai
-langchain-nvidia-ai-endpoints==0.1.6
+plangchain-nvidia-ai-endpoints==0.1.6
     # via scrapegraphai
 langchain-openai==0.1.17
     # via scrapegraphai
+
 langchain-text-splitters==0.2.2
     # via langchain
 langsmith==0.1.93
@@ -291,6 +296,7 @@ markdown-it-py==3.0.0
 markupsafe==2.1.5
     # via jinja2
 matplotlib==3.9.1
+
     # via burr
 mccabe==0.7.0
     # via pylint

requirements.lock

@@ -127,6 +127,7 @@ greenlet==3.0.3
     # via playwright
     # via sqlalchemy
 groq==0.9.0
+
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
     # via google-cloud-resource-manager
@@ -174,6 +175,7 @@ langchain==0.2.10
 langchain-anthropic==0.1.20
     # via scrapegraphai
 langchain-aws==0.1.12
+
     # via scrapegraphai
 langchain-core==0.2.22
     # via langchain
@@ -189,6 +191,7 @@ langchain-core==0.2.22
 langchain-fireworks==0.1.5
     # via scrapegraphai
 langchain-google-genai==1.0.8
+
     # via scrapegraphai
 langchain-google-vertexai==1.0.7
     # via scrapegraphai
@@ -198,6 +201,7 @@ langchain-nvidia-ai-endpoints==0.1.6
     # via scrapegraphai
 langchain-openai==0.1.17
     # via scrapegraphai
+
 langchain-text-splitters==0.2.2
     # via langchain
 langsmith==0.1.93
@@ -214,6 +218,7 @@ multidict==6.0.5
     # via yarl
 multiprocess==0.70.16
     # via mpire
+
 numpy==1.26.4
     # via faiss-cpu
     # via langchain
@@ -322,6 +327,7 @@ soupsieve==2.5
 sqlalchemy==2.0.31
     # via langchain
 tenacity==8.5.0
+
     # via langchain
     # via langchain-core
 tiktoken==0.7.0

scrapegraphai/graphs/abstract_graph.py

@@ -150,6 +150,10 @@ def _create_llm(self, llm_config: dict, chat=False) -> object:
 
         # If model instance is passed directly instead of the model details
         if "model_instance" in llm_params:
+            try:
+               self.model_token = llm_params["model_tokens"]
+            except KeyError as exc:
+               raise KeyError("model_tokens not specified") from exc
             return llm_params["model_instance"]
 
         # Instantiate the language model based on the model name

scrapegraphai/nodes/parse_node.py

@@ -94,4 +94,4 @@ def execute(self, state: dict) -> dict:
 
         state.update({self.output[0]: chunks})
 
-        return state
+        return state

scrapegraphai/telemetry/telemetry.py

@@ -91,7 +91,7 @@ def _check_config_and_environ_for_telemetry_flag(
     "python_version": f"{platform.python_version()}/{platform.python_implementation()}",
     "distinct_id": g_anonymous_id,
     "scrapegraphai_version": VERSION,
-    "telemetry_version": "0.0.1",
+    "telemetry_version": "0.0.3",
 }
 
 
@@ -156,7 +156,7 @@ def log_event(event: str, properties: Dict[str, any]):
         send_event_json(event_json)
 
 
-def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None):
+def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, llm_model: str, embedder_model: str, source_type: str, execution_time: float, response: dict = None, error_node: str = None, exception: str = None, total_tokens: int = None, is_library=True):
     properties = {
         "graph_name": graph_name,
         "source": source,
@@ -170,6 +170,7 @@ def log_graph_execution(graph_name: str, source: str, prompt:str, schema:dict, l
         "error_node": error_node,
         "exception": exception,
         "total_tokens": total_tokens,
+        "is_library": is_library
     }
     log_event("graph_execution", properties)