feat/drop langchain as dependency for embedders (#146)

* drop langchain as dependency for embedders

* bump changelog

* fix voyageai unit test

* swap deprecated .json() with .model_dump_json()

* swap deprecated .dict() with .model_dump()

* fix huggingface embedder

* remove import from voyageai method

* fix vertexai unit test

* Add vertexai to test deps to support unit test

* fix bedrock default model

* fix vertexai embedder

* fix vertexai unit test

* Add back in mixedbreadai deps

Author: rbiseck3

CHANGELOG.md

@@ -1,10 +1,11 @@
-## 0.0.22-dev2
+## 0.0.22
 
 ### Enhancements
 
 * **Add documentation for developing sources/destinations**
 * **Leverage `uv` for pip compile**
 * **Use incoming fsspec data to populate metadata** Rather than make additional calls to collect metadata after initial file list, use connector-specific data to populate the metadata. 
+* **Drop langchain as dependency for embedders**
 
 ## 0.0.21
 

requirements/embed/aws-bedrock.in

@@ -1,5 +1,3 @@
 -c ../common/constraints.txt
--c ../common/constraints.txt
 
 boto3
-langchain-community

requirements/embed/aws-bedrock.txt

@@ -1,159 +1,23 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile ./embed/aws-bedrock.in --output-file ./embed/aws-bedrock.txt --no-strip-extras --python-version 3.9
-aiohappyeyeballs==2.4.3
-    # via aiohttp
-aiohttp==3.10.8
-    # via
-    #   langchain
-    #   langchain-community
-aiosignal==1.3.1
-    # via aiohttp
-annotated-types==0.7.0
-    # via pydantic
-anyio==4.6.0
-    # via httpx
-async-timeout==4.0.3
-    # via
-    #   aiohttp
-    #   langchain
-attrs==24.2.0
-    # via aiohttp
+#    uv pip compile aws-bedrock.in --output-file aws-bedrock.txt --no-strip-extras
 boto3==1.34.131
-    # via -r ./embed/aws-bedrock.in
+    # via -r aws-bedrock.in
 botocore==1.34.131
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   boto3
     #   s3transfer
-certifi==2024.8.30
-    # via
-    #   httpcore
-    #   httpx
-    #   requests
-charset-normalizer==3.3.2
-    # via requests
-dataclasses-json==0.6.7
-    # via langchain-community
-exceptiongroup==1.2.2
-    # via anyio
-frozenlist==1.4.1
-    # via
-    #   aiohttp
-    #   aiosignal
-h11==0.14.0
-    # via httpcore
-httpcore==1.0.6
-    # via httpx
-httpx==0.27.2
-    # via langsmith
-idna==3.10
-    # via
-    #   anyio
-    #   httpx
-    #   requests
-    #   yarl
 jmespath==1.0.1
     # via
     #   boto3
     #   botocore
-jsonpatch==1.33
-    # via langchain-core
-jsonpointer==3.0.0
-    # via jsonpatch
-langchain==0.3.1
-    # via langchain-community
-langchain-community==0.3.1
-    # via
-    #   -c ./embed/../common/constraints.txt
-    #   -r ./embed/aws-bedrock.in
-langchain-core==0.3.7
-    # via
-    #   langchain
-    #   langchain-community
-    #   langchain-text-splitters
-langchain-text-splitters==0.3.0
-    # via langchain
-langsmith==0.1.129
-    # via
-    #   langchain
-    #   langchain-community
-    #   langchain-core
-marshmallow==3.22.0
-    # via dataclasses-json
-multidict==6.1.0
-    # via
-    #   aiohttp
-    #   yarl
-mypy-extensions==1.0.0
-    # via typing-inspect
-numpy==1.26.4
-    # via
-    #   -c ./embed/../common/constraints.txt
-    #   langchain
-    #   langchain-community
-orjson==3.10.7
-    # via langsmith
-packaging==23.2
-    # via
-    #   -c ./embed/../common/constraints.txt
-    #   langchain-core
-    #   marshmallow
-pydantic==2.9.2
-    # via
-    #   langchain
-    #   langchain-core
-    #   langsmith
-    #   pydantic-settings
-pydantic-core==2.23.4
-    # via pydantic
-pydantic-settings==2.5.2
-    # via langchain-community
 python-dateutil==2.9.0.post0
     # via botocore
-python-dotenv==1.0.1
-    # via pydantic-settings
-pyyaml==6.0.2
-    # via
-    #   langchain
-    #   langchain-community
-    #   langchain-core
-requests==2.32.3
-    # via
-    #   langchain
-    #   langchain-community
-    #   langsmith
 s3transfer==0.10.2
     # via boto3
 six==1.16.0
     # via python-dateutil
-sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
-sqlalchemy==2.0.35
-    # via
-    #   langchain
-    #   langchain-community
-tenacity==8.5.0
-    # via
-    #   langchain
-    #   langchain-community
-    #   langchain-core
-typing-extensions==4.12.2
-    # via
-    #   anyio
-    #   langchain-core
-    #   multidict
-    #   pydantic
-    #   pydantic-core
-    #   sqlalchemy
-    #   typing-inspect
-typing-inspect==0.9.0
-    # via dataclasses-json
 urllib3==1.26.20
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   botocore
-    #   requests
-yarl==1.13.1
-    # via aiohttp

requirements/embed/huggingface.in

@@ -1,5 +1,3 @@
 -c ../common/constraints.txt
--c ../common/constraints.txt
-
 
-langchain-huggingface
+sentence-transformers

requirements/embed/huggingface.txt

@@ -1,59 +1,30 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile ./embed/huggingface.in --output-file ./embed/huggingface.txt --no-strip-extras --python-version 3.9
-annotated-types==0.7.0
-    # via pydantic
-anyio==4.6.0
-    # via httpx
+#    uv pip compile huggingface.in --output-file huggingface.txt --no-strip-extras
 certifi==2024.8.30
-    # via
-    #   httpcore
-    #   httpx
-    #   requests
+    # via requests
 charset-normalizer==3.3.2
     # via requests
-exceptiongroup==1.2.2
-    # via anyio
 filelock==3.16.1
     # via
     #   huggingface-hub
     #   torch
     #   transformers
 fsspec==2024.5.0
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   huggingface-hub
     #   torch
-h11==0.14.0
-    # via httpcore
-httpcore==1.0.6
-    # via httpx
-httpx==0.27.2
-    # via langsmith
 huggingface-hub==0.25.1
     # via
-    #   langchain-huggingface
     #   sentence-transformers
     #   tokenizers
     #   transformers
 idna==3.10
-    # via
-    #   anyio
-    #   httpx
-    #   requests
+    # via requests
 jinja2==3.1.4
     # via torch
 joblib==1.4.2
     # via scikit-learn
-jsonpatch==1.33
-    # via langchain-core
-jsonpointer==3.0.0
-    # via jsonpatch
-langchain-core==0.3.7
-    # via langchain-huggingface
-langchain-huggingface==0.1.0
-    # via -r ./embed/huggingface.in
-langsmith==0.1.129
-    # via langchain-core
 markupsafe==2.1.5
     # via jinja2
 mpmath==1.3.0
@@ -62,37 +33,26 @@ networkx==3.2.1
     # via torch
 numpy==1.26.4
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   scikit-learn
     #   scipy
     #   transformers
-orjson==3.10.7
-    # via langsmith
 packaging==23.2
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   huggingface-hub
-    #   langchain-core
     #   transformers
 pillow==10.4.0
     # via sentence-transformers
-pydantic==2.9.2
-    # via
-    #   langchain-core
-    #   langsmith
-pydantic-core==2.23.4
-    # via pydantic
 pyyaml==6.0.2
     # via
     #   huggingface-hub
-    #   langchain-core
     #   transformers
 regex==2024.9.11
     # via transformers
 requests==2.32.3
     # via
     #   huggingface-hub
-    #   langsmith
     #   transformers
 safetensors==0.4.5
     # via transformers
@@ -103,21 +63,14 @@ scipy==1.13.1
     #   scikit-learn
     #   sentence-transformers
 sentence-transformers==3.1.1
-    # via langchain-huggingface
-sniffio==1.3.1
-    # via
-    #   anyio
-    #   httpx
+    # via -r huggingface.in
 sympy==1.13.3
     # via torch
-tenacity==8.5.0
-    # via langchain-core
 threadpoolctl==3.5.0
     # via scikit-learn
 tokenizers==0.19.1
     # via
-    #   -c ./embed/../common/constraints.txt
-    #   langchain-huggingface
+    #   -c ../common/constraints.txt
     #   transformers
 torch==2.4.1
     # via sentence-transformers
@@ -127,18 +80,12 @@ tqdm==4.66.5
     #   sentence-transformers
     #   transformers
 transformers==4.44.2
-    # via
-    #   langchain-huggingface
-    #   sentence-transformers
+    # via sentence-transformers
 typing-extensions==4.12.2
     # via
-    #   anyio
     #   huggingface-hub
-    #   langchain-core
-    #   pydantic
-    #   pydantic-core
     #   torch
 urllib3==1.26.20
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   requests

requirements/embed/mixedbreadai.in

@@ -1,4 +1,3 @@
 -c ../common/constraints.txt
--c ../common/constraints.txt
 
 mixedbread-ai

requirements/embed/mixedbreadai.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile ./embed/mixedbreadai.in --output-file ./embed/mixedbreadai.txt --no-strip-extras --python-version 3.9
+#    uv pip compile mixedbreadai.in --output-file mixedbreadai.txt --no-strip-extras
 annotated-types==0.7.0
     # via pydantic
 anyio==4.6.0
@@ -21,7 +21,7 @@ idna==3.10
     #   anyio
     #   httpx
 mixedbread-ai==2.2.6
-    # via -r ./embed/mixedbreadai.in
+    # via -r mixedbreadai.in
 pydantic==2.9.2
     # via mixedbread-ai
 pydantic-core==2.23.4

requirements/embed/octoai.in

@@ -1,5 +1,4 @@
 -c ../common/constraints.txt
--c ../common/constraints.txt
 
 openai
 tiktoken

requirements/embed/octoai.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile ./embed/octoai.in --output-file ./embed/octoai.txt --no-strip-extras --python-version 3.9
+#    uv pip compile octoai.in --output-file octoai.txt --no-strip-extras
 annotated-types==0.7.0
     # via pydantic
 anyio==4.6.0
@@ -31,7 +31,7 @@ idna==3.10
 jiter==0.5.0
     # via openai
 openai==1.51.0
-    # via -r ./embed/octoai.in
+    # via -r octoai.in
 pydantic==2.9.2
     # via openai
 pydantic-core==2.23.4
@@ -46,7 +46,7 @@ sniffio==1.3.1
     #   httpx
     #   openai
 tiktoken==0.7.0
-    # via -r ./embed/octoai.in
+    # via -r octoai.in
 tqdm==4.66.5
     # via openai
 typing-extensions==4.12.2
@@ -57,5 +57,5 @@ typing-extensions==4.12.2
     #   pydantic-core
 urllib3==1.26.20
     # via
-    #   -c ./embed/../common/constraints.txt
+    #   -c ../common/constraints.txt
     #   requests