refactoring of folders

Author: VinciGit00

examples/extras/screenshot_scaping.py

@@ -0,0 +1,26 @@
+from scrapegraphai.utils.screenshot_scraping import take_screenshot, select_area_with_opencv, crop_image, detect_text
+import asyncio
+
+# STEP 1: Take a screenshot
+image = asyncio.run(take_screenshot(
+    url="https://colab.google/",
+    save_path="Savedscreenshots/test_image.jpeg",
+    quality = 50
+))
+
+# STEP 2 (Optional): Select an area of the image which you want to use for text detection.
+LEFT, TOP, RIGHT, BOTTOM = select_area_with_opencv(image)
+print("LEFT: ", LEFT, " TOP: ", TOP, " RIGHT: ", RIGHT, " BOTTOM: ", BOTTOM)
+
+# STEP 3 (Optional): Crop the image.
+# Note: If any of the coordinates (LEFT, TOP, RIGHT, BOTTOM) is None, it will be set to the corresponding edge of the image.
+cropped_image = crop_image(image, LEFT=LEFT, RIGHT=RIGHT,TOP=TOP,BOTTOM=BOTTOM)
+
+# STEP 4: Detect text
+text = detect_text(
+    cropped_image,          # The image to detect text from
+    languages = ["en"]       # The languages to detect text in
+)
+
+print("DETECTED TEXT: ")
+print(text)

pyproject.toml

@@ -88,6 +88,13 @@ more-browser-options = [
     "browserbase>=0.3.0",
 ]
 
+# Group 4: Surya Library
+screenshot_scraper = [
+    "surya-ocr>=0.4.5",
+    "matplotlib>=3.7.2",
+    "ipywidgets>=8.1.0"
+]
+
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"

requirements-dev.lock

@@ -29,6 +29,8 @@ anyio==4.4.0
     # via starlette
 astroid==3.2.4
     # via pylint
+asttokens==2.4.1
+    # via stack-data
 async-timeout==4.0.3
     # via aiohttp
     # via langchain
@@ -64,12 +66,16 @@ click==8.1.7
     # via burr
     # via streamlit
     # via uvicorn
+comm==0.2.2
+    # via ipywidgets
 contourpy==1.2.1
     # via matplotlib
 cycler==0.12.1
     # via matplotlib
 dataclasses-json==0.6.7
     # via langchain-community
+decorator==5.1.1
+    # via ipython
 dill==0.3.8
     # via multiprocess
     # via pylint
@@ -79,7 +85,10 @@ docutils==0.19
     # via sphinx
 exceptiongroup==1.2.2
     # via anyio
+    # via ipython
     # via pytest
+executing==2.0.1
+    # via stack-data
 faiss-cpu==1.8.0.post1
     # via scrapegraphai
 fastapi==0.112.0
@@ -88,6 +97,10 @@ fastapi-pagination==0.12.26
     # via burr
 filelock==3.15.4
     # via huggingface-hub
+    # via torch
+    # via transformers
+filetype==1.2.0
+    # via surya-ocr
 fonttools==4.53.1
     # via matplotlib
 free-proxy==1.1.1
@@ -97,6 +110,9 @@ frozenlist==1.4.1
     # via aiosignal
 fsspec==2024.6.1
     # via huggingface-hub
+    # via torch
+ftfy==6.2.3
+    # via surya-ocr
 furo==2024.5.6
     # via scrapegraphai
 gitdb==4.0.11
@@ -152,6 +168,7 @@ httpx-sse==0.4.0
     # via langchain-mistralai
 huggingface-hub==0.24.5
     # via tokenizers
+    # via transformers
 idna==3.7
     # via anyio
     # via httpx
@@ -165,13 +182,20 @@ importlib-resources==6.4.0
     # via matplotlib
 iniconfig==2.0.0
     # via pytest
+ipython==8.18.1
+    # via ipywidgets
+ipywidgets==8.1.5
+    # via scrapegraphai
 isort==5.13.2
     # via pylint
+jedi==0.19.1
+    # via ipython
 jinja2==3.1.4
     # via altair
     # via burr
     # via pydeck
     # via sphinx
+    # via torch
 jiter==0.5.0
     # via openai
 jmespath==1.0.1
@@ -185,6 +209,8 @@ jsonschema==4.23.0
     # via altair
 jsonschema-specifications==2023.12.1
     # via jsonschema
+jupyterlab-widgets==3.0.13
+    # via ipywidgets
 kiwisolver==1.4.5
     # via matplotlib
 langchain==0.2.14
@@ -226,6 +252,9 @@ marshmallow==3.21.3
     # via dataclasses-json
 matplotlib==3.9.1.post1
     # via burr
+    # via scrapegraphai
+matplotlib-inline==0.1.7
+    # via ipython
 mccabe==0.7.0
     # via pylint
 mdurl==0.1.2
@@ -234,6 +263,8 @@ minify-html==0.15.0
     # via scrapegraphai
 mpire==2.10.2
     # via semchunk
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
@@ -243,21 +274,27 @@ mypy-extensions==1.0.0
     # via typing-inspect
 narwhals==1.3.0
     # via altair
+networkx==3.2.1
+    # via torch
 numpy==1.26.4
     # via contourpy
     # via faiss-cpu
     # via langchain
     # via langchain-aws
     # via langchain-community
     # via matplotlib
+    # via opencv-python
     # via pandas
     # via pyarrow
     # via pydeck
     # via sf-hamilton
     # via streamlit
+    # via transformers
 openai==1.40.3
     # via burr
     # via langchain-openai
+opencv-python==4.10.0.84
+    # via surya-ocr
 orjson==3.10.7
     # via langsmith
 packaging==24.1
@@ -270,20 +307,28 @@ packaging==24.1
     # via pytest
     # via sphinx
     # via streamlit
+    # via transformers
 pandas==2.2.2
     # via scrapegraphai
     # via sf-hamilton
     # via streamlit
+parso==0.8.4
+    # via jedi
+pexpect==4.9.0
+    # via ipython
 pillow==10.4.0
     # via matplotlib
     # via streamlit
+    # via surya-ocr
 platformdirs==4.2.2
     # via pylint
 playwright==1.45.1
     # via scrapegraphai
     # via undetected-playwright
 pluggy==1.5.0
     # via pytest
+prompt-toolkit==3.0.47
+    # via ipython
 proto-plus==1.24.0
     # via google-ai-generativelanguage
     # via google-api-core
@@ -295,6 +340,10 @@ protobuf==4.25.4
     # via grpcio-status
     # via proto-plus
     # via streamlit
+ptyprocess==0.7.0
+    # via pexpect
+pure-eval==0.2.3
+    # via stack-data
 pyarrow==17.0.0
     # via streamlit
 pyasn1==0.6.0
@@ -311,21 +360,28 @@ pydantic==2.8.2
     # via langchain-core
     # via langsmith
     # via openai
+    # via pydantic-settings
+    # via surya-ocr
 pydantic-core==2.20.1
     # via pydantic
+pydantic-settings==2.4.0
+    # via surya-ocr
 pydeck==0.9.1
     # via streamlit
 pyee==11.1.0
     # via playwright
 pygments==2.18.0
     # via furo
+    # via ipython
     # via mpire
     # via rich
     # via sphinx
 pylint==3.2.6
 pyparsing==3.1.2
     # via httplib2
     # via matplotlib
+pypdfium2==4.30.0
+    # via surya-ocr
 pytest==8.0.0
     # via pytest-mock
 pytest-mock==3.14.0
@@ -334,19 +390,23 @@ python-dateutil==2.9.0.post0
     # via matplotlib
     # via pandas
 python-dotenv==1.0.1
+    # via pydantic-settings
     # via scrapegraphai
+    # via surya-ocr
 pytz==2024.1
     # via pandas
 pyyaml==6.0.2
     # via huggingface-hub
     # via langchain
     # via langchain-community
     # via langchain-core
+    # via transformers
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
 regex==2024.7.24
     # via tiktoken
+    # via transformers
 requests==2.32.3
     # via burr
     # via free-proxy
@@ -358,6 +418,7 @@ requests==2.32.3
     # via sphinx
     # via streamlit
     # via tiktoken
+    # via transformers
 rich==13.7.1
     # via streamlit
 rpds-py==0.20.0
@@ -367,11 +428,14 @@ rsa==4.9
     # via google-auth
 s3transfer==0.10.2
     # via boto3
+safetensors==0.4.4
+    # via transformers
 semchunk==2.2.0
     # via scrapegraphai
 sf-hamilton==1.73.1
     # via burr
 six==1.16.0
+    # via asttokens
     # via python-dateutil
 smmap==5.0.1
     # via gitdb
@@ -404,10 +468,18 @@ sphinxcontrib-serializinghtml==2.0.0
 sqlalchemy==2.0.32
     # via langchain
     # via langchain-community
+stack-data==0.6.3
+    # via ipython
 starlette==0.37.2
     # via fastapi
 streamlit==1.37.1
     # via burr
+surya-ocr==0.5.0
+    # via scrapegraphai
+sympy==1.13.2
+    # via torch
+tabulate==0.9.0
+    # via surya-ocr
 tenacity==8.5.0
     # via langchain
     # via langchain-community
@@ -418,13 +490,16 @@ tiktoken==0.7.0
     # via scrapegraphai
 tokenizers==0.19.1
     # via langchain-mistralai
+    # via transformers
 toml==0.10.2
     # via streamlit
 tomli==2.0.1
     # via pylint
     # via pytest
 tomlkit==0.13.0
     # via pylint
+torch==2.4.0
+    # via surya-ocr
 tornado==6.4.1
     # via streamlit
 tqdm==4.66.5
@@ -434,6 +509,14 @@ tqdm==4.66.5
     # via openai
     # via scrapegraphai
     # via semchunk
+    # via transformers
+traitlets==5.14.3
+    # via comm
+    # via ipython
+    # via ipywidgets
+    # via matplotlib-inline
+transformers==4.44.2
+    # via surya-ocr
 typing-extensions==4.12.2
     # via altair
     # via anyio
@@ -442,6 +525,7 @@ typing-extensions==4.12.2
     # via fastapi-pagination
     # via google-generativeai
     # via huggingface-hub
+    # via ipython
     # via langchain-core
     # via openai
     # via pydantic
@@ -452,6 +536,7 @@ typing-extensions==4.12.2
     # via sqlalchemy
     # via starlette
     # via streamlit
+    # via torch
     # via typing-inspect
     # via uvicorn
 typing-inspect==0.9.0
@@ -468,6 +553,11 @@ urllib3==1.26.19
     # via requests
 uvicorn==0.30.5
     # via burr
+wcwidth==0.2.13
+    # via ftfy
+    # via prompt-toolkit
+widgetsnbextension==4.0.13
+    # via ipywidgets
 yarl==1.9.4
     # via aiohttp
 zipp==3.20.0