drop reference to langchain (#152)

rbiseck3 · web-flow · commit 277c99344ad3 · 2024-10-03T16:21:05.000-04:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.0.23
+
+### Fixes
+
+* **Remove check for langchain dependency in embedders**
+
 ## 0.0.22
 
 ### Enhancements
diff --git a/docs/README.md b/docs/README.md
@@ -146,7 +146,7 @@ if __name__ == "__main__":
         chunker_config=ChunkerConfig(
             chunking_strategy="basic",
         ),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         
         destination_connection_config=ChromaConnectionConfig(
             access_config=ChromaAccessConfig(settings=None, headers=None),
diff --git a/requirements/common/constraints.txt b/requirements/common/constraints.txt
@@ -15,16 +15,13 @@ urllib3<1.27
 botocore<1.34.132
 # TODO: Constriant due to both 8.5.0 and 8.4.0 being installed during pip-compile
 importlib-metadata>=8.5.0
-# TODO: Constraint due to langchain, remove when that gets updated:
-packaging<24.0
 # TODO: Constraint due to boto, with python before 3.10 not requiring openssl 1.1.1, remove when that gets
 # updated or we drop support for 3.9
 urllib3<1.27
 unstructured-client>= 0.25.8
 fsspec==2024.5.0
 # python 3.12 support
 wrapt>=1.14.0
-langchain-community>=0.2.5
 # NOTE(robinson): chroma was pinned to importlib-metadata>=7.1.0 but 7.1.0 was installed
 # instead of 7.2.0. Need to investigate
 importlib-metadata==7.1.0
diff --git a/test_e2e/dest/astradb.sh b/test_e2e/dest/astradb.sh
@@ -56,7 +56,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --chunking-strategy by_title \
   --chunk-max-characters 1500 \
   --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   astradb \
   --token "$ASTRA_DB_APPLICATION_TOKEN" \
   --api-endpoint "$ASTRA_DB_API_ENDPOINT" \
diff --git a/test_e2e/dest/azure-cognitive-search.sh b/test_e2e/dest/azure-cognitive-search.sh
@@ -86,7 +86,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --chunk-max-characters 2500 \
   --chunk-multipage-sections \
   --no-chunk-include-orig-elements \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   azure-cognitive-search \
   --key "$AZURE_SEARCH_API_KEY" \
   --endpoint "$AZURE_SEARCH_ENDPOINT" \
diff --git a/test_e2e/dest/chroma.sh b/test_e2e/dest/chroma.sh
@@ -45,7 +45,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --chunking-strategy by_title \
   --chunk-max-characters 1500 \
   --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   chroma \
   --host "localhost" \
   --port 8000 \
diff --git a/test_e2e/dest/couchbase.sh b/test_e2e/dest/couchbase.sh
@@ -67,7 +67,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --chunking-strategy by_title \
   --chunk-max-characters 1500 \
   --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   couchbase \
   --connection-string "$CB_CONN_STR" \
   --bucket "$CB_BUCKET" \
diff --git a/test_e2e/dest/elasticsearch.sh b/test_e2e/dest/elasticsearch.sh
@@ -50,7 +50,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --chunk-new-after-n-chars 2500 \
   --chunk-max-characters 38000 \
   --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   elasticsearch \
   --hosts http://localhost:9200 \
   --index-name ingest-test-destination \
diff --git a/test_e2e/dest/kafka-local.sh b/test_e2e/dest/kafka-local.sh
@@ -49,7 +49,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --chunk-new-after-n-chars 2500 \
   --chunk-max-characters 38000 \
   --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   kafka \
   --topic "$KAFKA_TOPIC" \
   --bootstrap-server "$KAFKA_BOOTSTRAP_SERVER" \
diff --git a/test_e2e/dest/kdbai.sh b/test_e2e/dest/kdbai.sh
@@ -48,7 +48,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --reprocess \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   kdbai \
   --table-name "unstructured_test" \
   --batch-size 100
diff --git a/test_e2e/dest/milvus.sh b/test_e2e/dest/milvus.sh
@@ -48,7 +48,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   milvus \
   --uri $MILVUS_URI \
   --db-name $DB_NAME \
diff --git a/test_e2e/dest/mongodb.sh b/test_e2e/dest/mongodb.sh
@@ -56,7 +56,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --reprocess \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   mongodb \
   --uri "$MONGODB_URI" \
   --database "$MONGODB_DATABASE_NAME" \
diff --git a/test_e2e/dest/opensearch.sh b/test_e2e/dest/opensearch.sh
@@ -43,7 +43,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --reprocess \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   opensearch \
   --hosts http://localhost:9247 \
   --index-name ingest-test-destination \
diff --git a/test_e2e/dest/pgvector.sh b/test_e2e/dest/pgvector.sh
@@ -42,7 +42,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --verbose \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   sql \
   --db-type "postgresql" \
   --username unstructured \
diff --git a/test_e2e/dest/pinecone.sh b/test_e2e/dest/pinecone.sh
@@ -96,7 +96,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --work-dir "$WORK_DIR" \
   --chunking-strategy by_title \
   --chunk-combine-text-under-n-chars 150 --chunk-new-after-n-chars 1500 --chunk-max-characters 2500 --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   pinecone \
   --api-key "$PINECONE_API_KEY" \
   --index-name "$PINECONE_INDEX" \
diff --git a/test_e2e/dest/qdrant.sh b/test_e2e/dest/qdrant.sh
@@ -49,7 +49,7 @@ curl -X PUT \
     }
 }'
 
-EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"langchain-huggingface"}
+EMBEDDING_PROVIDER=${EMBEDDING_PROVIDER:-"huggingface"}
 
 PYTHONPATH=. ./unstructured_ingest/main.py \
   local \
@@ -62,7 +62,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --work-dir "$WORK_DIR" \
   --chunking-strategy by_title \
   --chunk-combine-text-under-n-chars 200 --chunk-new-after-n-chars 2500 --chunk-max-characters 38000 --chunk-multipage-sections \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "$EMBEDDING_PROVIDER" \
   qdrant \
   --collection-name "$COLLECTION_NAME" \
   --location "http://"$QDRANT_HOST \
diff --git a/test_e2e/dest/sharepoint-embed-cog-index.sh b/test_e2e/dest/sharepoint-embed-cog-index.sh
@@ -97,7 +97,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --permissions-tenant "$SHAREPOINT_PERMISSIONS_TENANT" \
   --path "Shared Documents" \
   --recursive \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   --chunking-strategy by_title \
   --chunk-multipage-sections \
   --work-dir "$WORK_DIR" \
diff --git a/test_e2e/dest/singlestore.sh b/test_e2e/dest/singlestore.sh
@@ -48,7 +48,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --reprocess \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   singlestore \
   --host $HOST \
   --user $USER \
diff --git a/test_e2e/dest/weaviate.sh b/test_e2e/dest/weaviate.sh
@@ -42,7 +42,7 @@ PYTHONPATH=. ./unstructured_ingest/main.py \
   --reprocess \
   --input-path example-docs/pdf/fake-memo.pdf \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface" \
+  --embedding-provider "huggingface" \
   weaviate \
   --host-url http://localhost:8080 \
   --class-name elements \
diff --git a/test_e2e/src/local-embed-bedrock.sh b/test_e2e/src/local-embed-bedrock.sh
@@ -37,7 +37,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-aws-bedrock" \
+  --embedding-provider "aws-bedrock" \
   --embedding-aws-access-key-id "$AWS_ACCESS_KEY_ID" \
   --embedding-aws-secret-access-key "$AWS_SECRET_ACCESS_KEY"
 
diff --git a/test_e2e/src/local-embed-vertexai.sh b/test_e2e/src/local-embed-vertexai.sh
@@ -38,7 +38,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-vertexai" \
+  --embedding-provider "vertexai" \
   --embedding-api-key "$GCP_INGEST_SERVICE_KEY" \
   --embedding-model-name "textembedding-gecko@001"
 
diff --git a/test_e2e/src/local-embed-voyageai.sh b/test_e2e/src/local-embed-voyageai.sh
@@ -38,7 +38,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-voyageai" \
+  --embedding-provider "voyageai" \
   --embedding-api-key "$VOYAGE_API_KEY" \
   --embedding-model-name "voyage-large-2"
 
diff --git a/test_e2e/src/local-embed.sh b/test_e2e/src/local-embed.sh
@@ -32,7 +32,7 @@ PYTHONPATH=${PYTHONPATH:-.} "$RUN_SCRIPT" \
   --reprocess \
   --input-path example-docs/book-war-and-peace-1p.txt \
   --work-dir "$WORK_DIR" \
-  --embedding-provider "langchain-huggingface"
+  --embedding-provider "huggingface"
 
 set +e
 
diff --git a/unstructured_ingest/__version__.py b/unstructured_ingest/__version__.py
@@ -1 +1 @@
-__version__ = "0.0.22"  # pragma: no cover
+__version__ = "0.0.23"  # pragma: no cover
diff --git a/unstructured_ingest/cli/interfaces.py b/unstructured_ingest/cli/interfaces.py
@@ -415,11 +415,11 @@ class CliEmbeddingConfig(EmbeddingConfig, CliMixin):
     @staticmethod
     def get_cli_options() -> t.List[click.Option]:
         embed_providers = [
-            "langchain-openai",
-            "langchain-huggingface",
-            "langchain-aws-bedrock",
-            "langchain-vertexai",
-            "langchain-voyageai",
+            "openai",
+            "huggingface",
+            "aws-bedrock",
+            "vertexai",
+            "voyageai",
             "octoai",
         ]
         options = [
diff --git a/unstructured_ingest/embed/__init__.py b/unstructured_ingest/embed/__init__.py
@@ -1,17 +0,0 @@
-from unstructured_ingest.embed.bedrock import BedrockEmbeddingEncoder
-from unstructured_ingest.embed.huggingface import HuggingFaceEmbeddingEncoder
-from unstructured_ingest.embed.mixedbreadai import MixedbreadAIEmbeddingEncoder
-from unstructured_ingest.embed.octoai import OctoAIEmbeddingEncoder
-from unstructured_ingest.embed.openai import OpenAIEmbeddingEncoder
-from unstructured_ingest.embed.vertexai import VertexAIEmbeddingEncoder
-from unstructured_ingest.embed.voyageai import VoyageAIEmbeddingEncoder
-
-EMBEDDING_PROVIDER_TO_CLASS_MAP = {
-    "langchain-openai": OpenAIEmbeddingEncoder,
-    "langchain-huggingface": HuggingFaceEmbeddingEncoder,
-    "langchain-aws-bedrock": BedrockEmbeddingEncoder,
-    "langchain-vertexai": VertexAIEmbeddingEncoder,
-    "langchain-voyageai": VoyageAIEmbeddingEncoder,
-    "mixedbread-ai": MixedbreadAIEmbeddingEncoder,
-    "octoai": OctoAIEmbeddingEncoder,
-}
diff --git a/unstructured_ingest/embed/vertexai.py b/unstructured_ingest/embed/vertexai.py
@@ -45,7 +45,7 @@ def register_application_credentials(self):
         extras="embed-vertexai",
     )
     def get_client(self) -> "TextEmbeddingModel":
-        """Creates a Langchain VertexAI python client to embed elements."""
+        """Creates a VertexAI python client to embed elements."""
         from vertexai.language_models import TextEmbeddingModel
 
         self.register_application_credentials()
diff --git a/unstructured_ingest/embed/voyageai.py b/unstructured_ingest/embed/voyageai.py
@@ -20,11 +20,11 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
     timeout_in_seconds: Optional[int] = None
 
     @requires_dependencies(
-        ["langchain", "langchain_voyageai"],
+        ["voyageai"],
         extras="embed-voyageai",
     )
     def get_client(self) -> "VoyageAIClient":
-        """Creates a Langchain VoyageAI python client to embed elements."""
+        """Creates a VoyageAI python client to embed elements."""
         from voyageai import Client as VoyageAIClient
 
         client = VoyageAIClient(
diff --git a/unstructured_ingest/interfaces.py b/unstructured_ingest/interfaces.py
@@ -204,14 +204,14 @@ def get_embedder(self) -> "BaseEmbeddingEncoder":
         if self.model_name:
             kwargs["model_name"] = self.model_name
         # TODO make this more dynamic to map to encoder configs
-        if self.provider == "langchain-openai":
+        if self.provider == "openai":
             from unstructured_ingest.embed.openai import (
                 OpenAIEmbeddingConfig,
                 OpenAIEmbeddingEncoder,
             )
 
             return OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-huggingface":
+        elif self.provider == "huggingface":
             from unstructured_ingest.embed.huggingface import (
                 HuggingFaceEmbeddingConfig,
                 HuggingFaceEmbeddingEncoder,
@@ -225,7 +225,7 @@ def get_embedder(self) -> "BaseEmbeddingEncoder":
             )
 
             return OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-aws-bedrock":
+        elif self.provider == "aws-bedrock":
             from unstructured_ingest.embed.bedrock import (
                 BedrockEmbeddingConfig,
                 BedrockEmbeddingEncoder,
@@ -238,14 +238,14 @@ def get_embedder(self) -> "BaseEmbeddingEncoder":
                     region_name=self.aws_region,
                 )
             )
-        elif self.provider == "langchain-vertexai":
+        elif self.provider == "vertexai":
             from unstructured_ingest.embed.vertexai import (
                 VertexAIEmbeddingConfig,
                 VertexAIEmbeddingEncoder,
             )
 
             return VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(**kwargs))
-        elif self.provider == "langchain-voyageai":
+        elif self.provider == "voyageai":
             from unstructured_ingest.embed.voyageai import (
                 VoyageAIEmbeddingConfig,
                 VoyageAIEmbeddingEncoder,
diff --git a/unstructured_ingest/v2/examples/airtable.py b/unstructured_ingest/v2/examples/airtable.py
@@ -39,6 +39,6 @@
         ),
         partitioner_config=PartitionerConfig(strategy="fast"),
         chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
     ).run()
diff --git a/unstructured_ingest/v2/examples/azure_cognitive_search.py b/unstructured_ingest/v2/examples/azure_cognitive_search.py
@@ -41,7 +41,7 @@
             chunking_strategy="by_title", chunk_include_orig_elements=False
         ),
         embedder_config=EmbedderConfig(
-            embedding_provider="langchain-openai", embedding_api_key=os.getenv("OPENAI_API_KEY")
+            embedding_provider="openai", embedding_api_key=os.getenv("OPENAI_API_KEY")
         ),
         destination_connection_config=AzureCognitiveSearchConnectionConfig(
             access_config=AzureCognitiveSearchAccessConfig(
diff --git a/unstructured_ingest/v2/examples/chroma.py b/unstructured_ingest/v2/examples/chroma.py
@@ -40,7 +40,7 @@
             chunk_max_characters=1500,
             chunk_multipage_sections=True,
         ),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=ChromaConnectionConfig(
             access_config=ChromaAccessConfig(settings=None, headers=None),
             host="localhost",
diff --git a/unstructured_ingest/v2/examples/couchbase.py b/unstructured_ingest/v2/examples/couchbase.py
@@ -39,7 +39,7 @@
             chunk_max_characters=1500,
             chunk_multipage_sections=True,
         ),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=CouchbaseConnectionConfig(
             access_config=CouchbaseAccessConfig(
                 connection_string="couchbase://localhost",
diff --git a/unstructured_ingest/v2/examples/databricks_volumes.py b/unstructured_ingest/v2/examples/databricks_volumes.py
@@ -39,7 +39,7 @@
             chunk_max_characters=1500,
             chunk_multipage_sections=True,
         ),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=DatabricksVolumesConnectionConfig(
             access_config=DatabricksVolumesAccessConfig(
                 username=os.environ["DATABRICKS_USERNAME"],
diff --git a/unstructured_ingest/v2/examples/elasticsearch.py b/unstructured_ingest/v2/examples/elasticsearch.py
@@ -38,7 +38,7 @@
         source_connection_config=LocalConnectionConfig(),
         partitioner_config=PartitionerConfig(strategy="fast"),
         chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=ElasticsearchConnectionConfig(
             access_config=ElasticsearchAccessConfig(password=os.getenv("ELASTIC_PASSWORD")),
             username=os.getenv("ELASTIC_USERNAME"),
diff --git a/unstructured_ingest/v2/examples/kdbai.py b/unstructured_ingest/v2/examples/kdbai.py
@@ -42,7 +42,7 @@
         source_connection_config=LocalConnectionConfig(),
         partitioner_config=PartitionerConfig(strategy="fast"),
         chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         destination_connection_config=KdbaiConnectionConfig(
             endpoint=os.environ["KDBAI_ENDPOINT"],
         ),
diff --git a/unstructured_ingest/v2/examples/local.py b/unstructured_ingest/v2/examples/local.py
@@ -31,6 +31,6 @@
         source_connection_config=LocalConnectionConfig(),
         partitioner_config=PartitionerConfig(strategy="fast"),
         chunker_config=ChunkerConfig(chunking_strategy="by_title"),
-        embedder_config=EmbedderConfig(embedding_provider="langchain-huggingface"),
+        embedder_config=EmbedderConfig(embedding_provider="huggingface"),
         uploader_config=LocalUploaderConfig(output_dir=str(output_path.resolve())),
     ).run()
diff --git a/unstructured_ingest/v2/examples/milvus.py b/unstructured_ingest/v2/examples/milvus.py
diff --git a/unstructured_ingest/v2/examples/mongodb.py b/unstructured_ingest/v2/examples/mongodb.py
diff --git a/unstructured_ingest/v2/examples/opensearch.py b/unstructured_ingest/v2/examples/opensearch.py
diff --git a/unstructured_ingest/v2/examples/pinecone.py b/unstructured_ingest/v2/examples/pinecone.py
diff --git a/unstructured_ingest/v2/examples/s3.py b/unstructured_ingest/v2/examples/s3.py
diff --git a/unstructured_ingest/v2/examples/salesforce.py b/unstructured_ingest/v2/examples/salesforce.py
diff --git a/unstructured_ingest/v2/examples/sharepoint.py b/unstructured_ingest/v2/examples/sharepoint.py
diff --git a/unstructured_ingest/v2/examples/singlestore.py b/unstructured_ingest/v2/examples/singlestore.py
diff --git a/unstructured_ingest/v2/examples/sql.py b/unstructured_ingest/v2/examples/sql.py
diff --git a/unstructured_ingest/v2/examples/weaviate.py b/unstructured_ingest/v2/examples/weaviate.py
diff --git a/unstructured_ingest/v2/processes/embedder.py b/unstructured_ingest/v2/processes/embedder.py
