refactor: remove redundant LangChain wrappers

Author: f-aguzzi

pyproject.toml

@@ -34,7 +34,8 @@ dependencies = [
     "undetected-playwright>=0.3.0",
     "semchunk>=1.0.1",
     "langchain-fireworks>=0.1.3",
-    "langchain-community>=0.2.9"
+    "langchain-community>=0.2.9",
+    "langchain-huggingface>=0.0.3",
 ]
 
 license = "MIT"

requirements-dev.lock

@@ -106,6 +106,8 @@ fastapi-pagination==0.12.26
     # via burr
 filelock==3.15.4
     # via huggingface-hub
+    # via torch
+    # via transformers
 fireworks-ai==0.14.0
     # via langchain-fireworks
 fonttools==4.53.1
@@ -117,6 +119,7 @@ frozenlist==1.4.1
     # via aiosignal
 fsspec==2024.6.1
     # via huggingface-hub
+    # via torch
 furo==2024.5.6
     # via scrapegraphai
 gitdb==4.0.11
@@ -180,6 +183,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -212,7 +216,10 @@ httpx==0.27.0
 httpx-sse==0.4.0
     # via fireworks-ai
 huggingface-hub==0.24.0
+    # via langchain-huggingface
+    # via sentence-transformers
     # via tokenizers
+    # via transformers
 idna==3.7
     # via anyio
     # via email-validator
@@ -235,11 +242,14 @@ jinja2==3.1.4
     # via fastapi
     # via pydeck
     # via sphinx
+    # via torch
 jiter==0.5.0
     # via anthropic
 jmespath==1.0.1
     # via boto3
     # via botocore
+joblib==1.4.2
+    # via scikit-learn
 jsonpatch==1.33
     # via langchain-core
 jsonpointer==3.0.0
@@ -268,6 +278,7 @@ langchain-core==0.2.22
     # via langchain-google-genai
     # via langchain-google-vertexai
     # via langchain-groq
+    # via langchain-huggingface
     # via langchain-nvidia-ai-endpoints
     # via langchain-openai
     # via langchain-text-splitters
@@ -279,6 +290,8 @@ langchain-google-vertexai==1.0.7
     # via scrapegraphai
 langchain-groq==0.1.6
     # via scrapegraphai
+langchain-huggingface==0.0.3
+    # via scrapegraphai
 langchain-nvidia-ai-endpoints==0.1.6
     # via scrapegraphai
 langchain-openai==0.1.17
@@ -309,13 +322,17 @@ minify-html==0.15.0
     # via scrapegraphai
 mpire==2.10.2
     # via semchunk
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
 multiprocess==0.70.16
     # via mpire
 mypy-extensions==1.0.0
     # via typing-inspect
+networkx==3.2.1
+    # via torch
 numpy==1.26.4
     # via altair
     # via contourpy
@@ -327,9 +344,13 @@ numpy==1.26.4
     # via pandas
     # via pyarrow
     # via pydeck
+    # via scikit-learn
+    # via scipy
+    # via sentence-transformers
     # via sf-hamilton
     # via shapely
     # via streamlit
+    # via transformers
 openai==1.37.0
     # via burr
     # via langchain-fireworks
@@ -348,6 +369,7 @@ packaging==24.1
     # via pytest
     # via sphinx
     # via streamlit
+    # via transformers
 pandas==2.2.2
     # via altair
     # via scrapegraphai
@@ -357,6 +379,7 @@ pillow==10.4.0
     # via fireworks-ai
     # via langchain-nvidia-ai-endpoints
     # via matplotlib
+    # via sentence-transformers
     # via streamlit
 platformdirs==4.2.2
     # via pylint
@@ -436,12 +459,14 @@ pyyaml==6.0.1
     # via langchain
     # via langchain-community
     # via langchain-core
+    # via transformers
     # via uvicorn
 referencing==0.35.1
     # via jsonschema
     # via jsonschema-specifications
 regex==2024.5.15
     # via tiktoken
+    # via transformers
 requests==2.32.3
     # via burr
     # via free-proxy
@@ -456,6 +481,7 @@ requests==2.32.3
     # via sphinx
     # via streamlit
     # via tiktoken
+    # via transformers
 rich==13.7.1
     # via streamlit
     # via typer
@@ -466,8 +492,17 @@ rsa==4.9
     # via google-auth
 s3transfer==0.10.2
     # via boto3
+safetensors==0.4.3
+    # via transformers
+scikit-learn==1.5.1
+    # via sentence-transformers
+scipy==1.13.1
+    # via scikit-learn
+    # via sentence-transformers
 semchunk==2.2.0
     # via scrapegraphai
+sentence-transformers==3.0.1
+    # via langchain-huggingface
 sf-hamilton==1.72.1
     # via burr
 shapely==2.0.5
@@ -513,16 +548,22 @@ starlette==0.37.2
     # via fastapi
 streamlit==1.36.0
     # via burr
+sympy==1.13.1
+    # via torch
 tenacity==8.5.0
     # via langchain
     # via langchain-community
     # via langchain-core
     # via streamlit
+threadpoolctl==3.5.0
+    # via scikit-learn
 tiktoken==0.7.0
     # via langchain-openai
     # via scrapegraphai
 tokenizers==0.19.1
     # via anthropic
+    # via langchain-huggingface
+    # via transformers
 toml==0.10.2
     # via streamlit
 tomli==2.0.1
@@ -532,6 +573,8 @@ tomlkit==0.13.0
     # via pylint
 toolz==0.12.1
     # via altair
+torch==2.2.2
+    # via sentence-transformers
 tornado==6.4.1
     # via streamlit
 tqdm==4.66.4
@@ -541,6 +584,11 @@ tqdm==4.66.4
     # via openai
     # via scrapegraphai
     # via semchunk
+    # via sentence-transformers
+    # via transformers
+transformers==4.43.3
+    # via langchain-huggingface
+    # via sentence-transformers
 typer==0.12.3
     # via fastapi-cli
 typing-extensions==4.12.2
@@ -562,6 +610,7 @@ typing-extensions==4.12.2
     # via sqlalchemy
     # via starlette
     # via streamlit
+    # via torch
     # via typer
     # via typing-inspect
     # via uvicorn