feat: finished basic version of deep scraper

Co-Authored-By: Matteo Vedovati <[email protected]>

Author: VinciGit00

examples/openai/depth_search_graph_openai.py

@@ -1,22 +1,28 @@
 """
 depth_search_graph_opeani example
 """
+import os
+from dotenv import load_dotenv
 from scrapegraphai.graphs import DepthSearchGraph
 
+load_dotenv()
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
 graph_config = {
     "llm": {
-        "api_key":"YOUR_API_KEY",
+        "api_key": openai_key,
         "model": "openai/gpt-4o-mini",
     },
     "verbose": True,
     "headless": False,
     "depth": 2,
-    "only_inside_links": True,
+    "only_inside_links": False,
 }
 
 search_graph = DepthSearchGraph(
     prompt="List me all the projects with their description",
-    source="https://perinim.github.io/projects/",
+    source="https://perinim.github.io",
     config=graph_config
 )
 

pyproject.toml

@@ -31,7 +31,9 @@ dependencies = [
     "google>=3.0.0",
     "langchain-ollama>=0.1.3",
     "semchunk==2.2.0",
-    "transformers==4.44.2"
+    "transformers==4.44.2",
+    "qdrant-client>=1.11.3",
+    "fastembed>=0.3.6"
 ]
 
 license = "MIT"
@@ -99,7 +101,7 @@ screenshot_scraper = [
     "pillow>=10.4.0",
 ]
 
-# Group 5: Faiss CPU
+# Group 5: qdrant
 qdrant = [
     "qdrant-client>=1.11.3",
     "fastembed>=0.3.6"

requirements-dev.lock

@@ -64,6 +64,8 @@ click==8.1.7
     # via burr
     # via streamlit
     # via uvicorn
+coloredlogs==15.0.1
+    # via onnxruntime
 contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
@@ -84,9 +86,13 @@ fastapi==0.112.0
     # via burr
 fastapi-pagination==0.12.26
     # via burr
+fastembed==0.3.6
+    # via scrapegraphai
 filelock==3.15.4
     # via huggingface-hub
     # via transformers
+flatbuffers==24.3.25
+    # via onnxruntime
 fonttools==4.53.1
     # via matplotlib
 free-proxy==1.1.1
@@ -132,11 +138,19 @@ greenlet==3.0.3
 grpcio==1.65.4
     # via google-api-core
     # via grpcio-status
+    # via grpcio-tools
+    # via qdrant-client
 grpcio-status==1.62.3
     # via google-api-core
+grpcio-tools==1.62.3
+    # via qdrant-client
 h11==0.14.0
     # via httpcore
     # via uvicorn
+h2==4.1.0
+    # via httpx
+hpack==4.0.0
+    # via h2
 html2text==2024.2.26
     # via scrapegraphai
 httpcore==1.0.5
@@ -149,11 +163,17 @@ httpx==0.27.0
     # via langsmith
     # via ollama
     # via openai
+    # via qdrant-client
 httpx-sse==0.4.0
     # via langchain-mistralai
 huggingface-hub==0.24.5
+    # via fastembed
     # via tokenizers
     # via transformers
+humanfriendly==10.0
+    # via coloredlogs
+hyperframe==6.0.1
+    # via h2
 idna==3.7
     # via anyio
     # via httpx
@@ -218,6 +238,7 @@ langsmith==0.1.121
     # via langchain-core
 loguru==0.7.2
     # via burr
+    # via fastembed
 lxml==5.3.0
     # via free-proxy
 markdown-it-py==3.0.0
@@ -236,8 +257,12 @@ minify-html==0.15.0
     # via scrapegraphai
 mistral-common==1.4.1
     # via scrapegraphai
+mmh3==4.1.0
+    # via fastembed
 mpire==2.10.2
     # via semchunk
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
@@ -249,19 +274,27 @@ narwhals==1.3.0
     # via altair
 numpy==1.26.4
     # via contourpy
+    # via fastembed
     # via langchain
     # via langchain-aws
     # via langchain-community
     # via matplotlib
+    # via onnx
+    # via onnxruntime
     # via opencv-python-headless
     # via pandas
     # via pyarrow
     # via pydeck
+    # via qdrant-client
     # via sf-hamilton
     # via streamlit
     # via transformers
 ollama==0.3.2
     # via langchain-ollama
+onnx==1.17.0
+    # via fastembed
+onnxruntime==1.19.2
+    # via fastembed
 openai==1.40.3
     # via burr
     # via langchain-openai
@@ -275,6 +308,7 @@ packaging==24.1
     # via langchain-core
     # via marshmallow
     # via matplotlib
+    # via onnxruntime
     # via pytest
     # via sphinx
     # via streamlit
@@ -284,6 +318,7 @@ pandas==2.2.2
     # via sf-hamilton
     # via streamlit
 pillow==10.4.0
+    # via fastembed
     # via matplotlib
     # via mistral-common
     # via streamlit
@@ -294,6 +329,8 @@ playwright==1.45.1
     # via undetected-playwright
 pluggy==1.5.0
     # via pytest
+portalocker==2.10.1
+    # via qdrant-client
 proto-plus==1.24.0
     # via google-ai-generativelanguage
     # via google-api-core
@@ -303,6 +340,9 @@ protobuf==4.25.4
     # via google-generativeai
     # via googleapis-common-protos
     # via grpcio-status
+    # via grpcio-tools
+    # via onnx
+    # via onnxruntime
     # via proto-plus
     # via streamlit
 pyarrow==17.0.0
@@ -326,6 +366,7 @@ pydantic==2.8.2
     # via mistral-common
     # via openai
     # via pydantic-settings
+    # via qdrant-client
 pydantic-core==2.20.1
     # via pydantic
 pydantic-settings==2.5.2
@@ -343,6 +384,8 @@ pylint==3.2.6
 pyparsing==3.1.2
     # via httplib2
     # via matplotlib
+pystemmer==2.2.0.1
+    # via fastembed
 pytest==8.0.0
     # via pytest-mock
 pytest-mock==3.14.0
@@ -361,6 +404,8 @@ pyyaml==6.0.2
     # via langchain-community
     # via langchain-core
     # via transformers
+qdrant-client==1.11.3
+    # via scrapegraphai
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
@@ -369,6 +414,7 @@ regex==2024.7.24
     # via transformers
 requests==2.32.3
     # via burr
+    # via fastembed
     # via free-proxy
     # via google-api-core
     # via huggingface-hub
@@ -395,6 +441,8 @@ semchunk==2.2.0
     # via scrapegraphai
 sentencepiece==0.2.0
     # via mistral-common
+setuptools==75.1.0
+    # via grpcio-tools
 sf-hamilton==1.73.1
     # via burr
 six==1.16.0
@@ -406,6 +454,7 @@ sniffio==1.3.1
     # via httpx
     # via openai
 snowballstemmer==2.2.0
+    # via fastembed
     # via sphinx
 soupsieve==2.5
     # via beautifulsoup4
@@ -434,6 +483,8 @@ starlette==0.37.2
     # via fastapi
 streamlit==1.37.1
     # via burr
+sympy==1.13.3
+    # via onnxruntime
 tenacity==8.5.0
     # via langchain
     # via langchain-community
@@ -444,6 +495,7 @@ tiktoken==0.7.0
     # via mistral-common
     # via scrapegraphai
 tokenizers==0.19.1
+    # via fastembed
     # via langchain-mistralai
     # via transformers
 toml==0.10.2
@@ -456,6 +508,7 @@ tomlkit==0.13.0
 tornado==6.4.1
     # via streamlit
 tqdm==4.66.5
+    # via fastembed
     # via google-generativeai
     # via huggingface-hub
     # via mpire
@@ -495,6 +548,7 @@ uritemplate==4.1.1
     # via google-api-python-client
 urllib3==1.26.19
     # via botocore
+    # via qdrant-client
     # via requests
 uvicorn==0.30.5
     # via burr