feat(burr): added burr integration in graphs and optional burr installation

Author: PeriniM

examples/openai/search_graph_burr.py

@@ -0,0 +1,50 @@
+"""
+Example of Search Graph
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SearchGraph
+from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
+load_dotenv()
+
+# ************************************************
+# Define the configuration for the graph
+# ************************************************
+
+openai_key = os.getenv("OPENAI_APIKEY")
+
+graph_config = {
+    "llm": {
+        "api_key": openai_key,
+        "model": "gpt-3.5-turbo",
+    },
+    "max_results": 2,
+    "verbose": True,
+    "burr_kwargs": {
+        "project_name": "search-graph-openai",
+    }
+}
+
+# ************************************************
+# Create the SearchGraph instance and run it
+# ************************************************
+
+search_graph = SearchGraph(
+    prompt="List me Chioggia's attractions.",
+    config=graph_config
+)
+
+result = search_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = search_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))
+
+# Save to json and csv
+convert_to_csv(result, "result")
+convert_to_json(result, "result")

examples/openai/burr_integration_openai.py renamed to examples/openai/smart_scraper_burr.py

@@ -29,14 +29,13 @@ dependencies = [
     "playwright==1.43.0",
     "google==3.0.0",
     "yahoo-search-py==0.3",
-    "burr[start]==0.17.1"
 ]
 
 license = "MIT"
 readme = "README.md"
-homepage = "https://scrapegraph-ai.readthedocs.io/"
+homepage = "https://scrapegraphai.com/"
 repository = "https://github.com/VinciGit00/Scrapegraph-ai"
-documentation = "https://scrapegraph-doc.onrender.com/"
+documentation = "https://scrapegraph-ai.readthedocs.io/en/latest/"
 keywords = [
     "scrapegraph",
     "scrapegraphai",
@@ -64,6 +63,10 @@ classifiers = [
 ]
 requires-python = ">= 3.9, < 3.12"
 
+[project.optional-dependencies]
+burr = ["burr[start]==0.18.0"]
+docs = ["sphinx==4.3.0", "sphinx-rtd-theme==1.0.0"]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
@@ -72,12 +75,7 @@ build-backend = "hatchling.build"
 managed = true
 dev-dependencies = [
     "pytest==8.0.0",
-    "pytest-mock==3.14.0"
-]
-
-[tool.rye.group.docs]
-optional = true
-
-[tool.rye.group.docs.dependencies]
-sphinx = "7.1.2"
-sphinx-rtd-theme = "2.0.0"
+    "pytest-mock==3.14.0",
+    "-e file:.[burr]",
+    "-e file:.[docs]",
+]

pyproject.toml

@@ -6,7 +6,6 @@
 #   features: []
 #   all-features: false
 #   with-sources: false
-#   generate-hashes: false
 
 -e file:.
 aiofiles==23.2.1
@@ -16,11 +15,13 @@ aiohttp==3.9.5
     # via langchain-community
 aiosignal==1.3.1
     # via aiohttp
+alabaster==0.7.16
+    # via sphinx
 altair==5.3.0
     # via streamlit
-annotated-types==0.6.0
+annotated-types==0.7.0
     # via pydantic
-anthropic==0.25.9
+anthropic==0.26.1
     # via langchain-anthropic
 anyio==4.3.0
     # via anthropic
@@ -36,17 +37,20 @@ attrs==23.2.0
     # via aiohttp
     # via jsonschema
     # via referencing
+babel==2.15.0
+    # via sphinx
 beautifulsoup4==4.12.3
     # via google
     # via scrapegraphai
 blinker==1.8.2
     # via streamlit
-boto3==1.34.105
+boto3==1.34.110
     # via langchain-aws
-botocore==1.34.105
+botocore==1.34.110
     # via boto3
     # via s3transfer
-burr==0.17.1
+burr==0.18.0
+    # via burr
     # via scrapegraphai
 cachetools==5.3.3
     # via google-auth
@@ -66,6 +70,7 @@ colorama==0.4.6
     # via click
     # via loguru
     # via pytest
+    # via sphinx
     # via tqdm
     # via uvicorn
 contourpy==1.2.1
@@ -83,6 +88,9 @@ distro==1.9.0
     # via openai
 dnspython==2.6.1
     # via email-validator
+docutils==0.17.1
+    # via sphinx
+    # via sphinx-rtd-theme
 email-validator==2.1.1
     # via fastapi
 exceptiongroup==1.2.1
@@ -106,15 +114,15 @@ free-proxy==1.1.1
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
-fsspec==2024.3.1
+fsspec==2024.5.0
     # via huggingface-hub
 gitdb==4.0.11
     # via gitpython
 gitpython==3.1.43
     # via streamlit
 google==3.0.0
     # via scrapegraphai
-google-ai-generativelanguage==0.6.3
+google-ai-generativelanguage==0.6.4
     # via google-generativeai
 google-api-core==2.19.0
     # via google-ai-generativelanguage
@@ -130,7 +138,7 @@ google-auth==2.29.0
     # via google-generativeai
 google-auth-httplib2==0.2.0
     # via google-api-python-client
-google-generativeai==0.5.3
+google-generativeai==0.5.4
     # via langchain-google-genai
 googleapis-common-protos==1.63.0
     # via google-api-core
@@ -141,9 +149,9 @@ graphviz==0.20.3
 greenlet==3.0.3
     # via playwright
     # via sqlalchemy
-groq==0.5.0
+groq==0.7.0
     # via langchain-groq
-grpcio==1.63.0
+grpcio==1.64.0
     # via google-api-core
     # via grpcio-status
 grpcio-status==1.62.2
@@ -166,21 +174,26 @@ httpx==0.27.0
     # via groq
     # via openai
     # via yahoo-search-py
-huggingface-hub==0.23.0
+huggingface-hub==0.23.1
     # via tokenizers
 idna==3.7
     # via anyio
     # via email-validator
     # via httpx
     # via requests
     # via yarl
+imagesize==1.4.1
+    # via sphinx
 iniconfig==2.0.0
     # via pytest
 jinja2==3.1.4
     # via altair
     # via burr
     # via fastapi
     # via pydeck
+    # via sphinx
+jiter==0.1.0
+    # via anthropic
 jmespath==1.0.1
     # via boto3
     # via botocore
@@ -218,9 +231,9 @@ langchain-groq==0.1.3
     # via scrapegraphai
 langchain-openai==0.1.6
     # via scrapegraphai
-langchain-text-splitters==0.0.1
+langchain-text-splitters==0.0.2
     # via langchain
-langsmith==0.1.57
+langsmith==0.1.60
     # via langchain
     # via langchain-community
     # via langchain-core
@@ -271,6 +284,7 @@ packaging==23.2
     # via marshmallow
     # via matplotlib
     # via pytest
+    # via sphinx
     # via streamlit
 pandas==2.2.2
     # via altair
@@ -322,6 +336,7 @@ pyee==11.1.0
     # via playwright
 pygments==2.18.0
     # via rich
+    # via sphinx
 pyparsing==3.1.2
     # via httplib2
     # via matplotlib
@@ -348,16 +363,17 @@ pyyaml==6.0.1
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
-regex==2024.5.10
+regex==2024.5.15
     # via tiktoken
-requests==2.31.0
+requests==2.32.2
     # via burr
     # via free-proxy
     # via google-api-core
     # via huggingface-hub
     # via langchain
     # via langchain-community
     # via langsmith
+    # via sphinx
     # via streamlit
     # via tiktoken
 rich==13.7.1
@@ -372,7 +388,7 @@ s3transfer==0.10.1
     # via boto3
 selectolax==0.3.21
     # via yahoo-search-py
-sf-hamilton==1.62.0
+sf-hamilton==1.63.0
     # via burr
 shellingham==1.5.4
     # via typer
@@ -386,8 +402,27 @@ sniffio==1.3.1
     # via groq
     # via httpx
     # via openai
+snowballstemmer==2.2.0
+    # via sphinx
 soupsieve==2.5
     # via beautifulsoup4
+sphinx==4.3.0
+    # via scrapegraphai
+    # via sphinx-rtd-theme
+sphinx-rtd-theme==1.0.0
+    # via scrapegraphai
+sphinxcontrib-applehelp==1.0.8
+    # via sphinx
+sphinxcontrib-devhelp==1.0.6
+    # via sphinx
+sphinxcontrib-htmlhelp==2.0.5
+    # via sphinx
+sphinxcontrib-jsmath==1.0.1
+    # via sphinx
+sphinxcontrib-qthelp==1.0.7
+    # via sphinx
+sphinxcontrib-serializinghtml==1.1.10
+    # via sphinx
 sqlalchemy==2.0.30
     # via langchain
     # via langchain-community
@@ -448,7 +483,7 @@ ujson==5.10.0
     # via fastapi
 uritemplate==4.1.1
     # via google-api-python-client
-urllib3==1.26.18
+urllib3==2.2.1
     # via botocore
     # via requests
     # via yahoo-search-py
@@ -467,3 +502,5 @@ yahoo-search-py==0.3
     # via scrapegraphai
 yarl==1.9.4
     # via aiohttp
+setuptools==70.0.0
+    # via sphinx