bugfx/allow for optional api keys in embedders (#479)

* allow for optional api keys in embedders

* upgrade deps

* fix import

* pin ES

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.17
+
+* **Support optional API keys for embedders**
+
 ## 1.0.16
 
 * **Add embedder config field descriptions**

requirements/connectors/elasticsearch.txt

@@ -1 +1 @@
-elasticsearch[async]
+elasticsearch[async]<9.0.0

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.16"  # pragma: no cover
+__version__ = "1.0.17"  # pragma: no cover

unstructured_ingest/embed/azure_openai.py

@@ -25,8 +25,9 @@ class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
     def get_client(self) -> "AzureOpenAI":
         from openai import AzureOpenAI
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         return AzureOpenAI(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,
         )
@@ -35,8 +36,9 @@ def get_client(self) -> "AzureOpenAI":
     def get_async_client(self) -> "AsyncAzureOpenAI":
         from openai import AsyncAzureOpenAI
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         return AsyncAzureOpenAI(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             api_version=self.api_version,
             azure_endpoint=self.azure_endpoint,
         )

unstructured_ingest/embed/octoai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from pydantic import Field, SecretStr
 
@@ -24,7 +24,7 @@
 
 
 class OctoAiEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
+    api_key: Optional[SecretStr] = Field(description="API key for OctoAI", default=None)
     embedder_model_name: str = Field(
         default="thenlper/gte-large", alias="model_name", description="octoai model name"
     )
@@ -77,7 +77,8 @@ def get_client(self) -> "OpenAI":
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return OpenAI(api_key=api_key, base_url=self.base_url)
 
     @requires_dependencies(
         ["openai", "tiktoken"],
@@ -87,7 +88,8 @@ def get_async_client(self) -> "AsyncOpenAI":
         """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncOpenAI(api_key=api_key, base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/embed/openai.py

@@ -24,7 +24,7 @@
 
 
 class OpenAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr = Field(description="API key for OpenAI")
+    api_key: Optional[SecretStr] = Field(description="API key for OpenAI", default=None)
     embedder_model_name: str = Field(
         default="text-embedding-ada-002", alias="model_name", description="OpenAI model name"
     )
@@ -88,13 +88,15 @@ def run_precheck(self) -> None:
     def get_client(self) -> "OpenAI":
         from openai import OpenAI
 
-        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return OpenAI(api_key=api_key, base_url=self.base_url)
 
     @requires_dependencies(["openai"], extras="openai")
     def get_async_client(self) -> "AsyncOpenAI":
         from openai import AsyncOpenAI
 
-        return AsyncOpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncOpenAI(api_key=api_key, base_url=self.base_url)
 
 
 @dataclass

unstructured_ingest/embed/togetherai.py

@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
 from pydantic import Field, SecretStr
 
@@ -20,7 +20,7 @@
 
 
 class TogetherAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr = Field(description="API key for Together AI")
+    api_key: Optional[SecretStr] = Field(description="API key for Together AI", default=None)
     embedder_model_name: str = Field(
         default="togethercomputer/m2-bert-80M-8k-retrieval",
         alias="model_name",
@@ -58,13 +58,15 @@ def run_precheck(self) -> None:
     def get_client(self) -> "Together":
         from together import Together
 
-        return Together(api_key=self.api_key.get_secret_value())
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return Together(api_key=api_key)
 
     @requires_dependencies(["together"], extras="togetherai")
     def get_async_client(self) -> "AsyncTogether":
         from together import AsyncTogether
 
-        return AsyncTogether(api_key=self.api_key.get_secret_value())
+        api_key = self.api_key.get_secret_value() if self.api_key else None
+        return AsyncTogether(api_key=api_key)
 
 
 @dataclass

unstructured_ingest/embed/voyageai.py

@@ -26,7 +26,7 @@ class VoyageAIEmbeddingConfig(EmbeddingConfig):
         le=128,
         description="Batch size for embedding requests. VoyageAI has a limit of 128.",
     )
-    api_key: SecretStr = Field(description="API key for VoyageAI")
+    api_key: Optional[SecretStr] = Field(description="API key for VoyageAI", default=None)
     embedder_model_name: str = Field(
         default="voyage-3", alias="model_name", description="VoyageAI model name"
     )
@@ -65,8 +65,9 @@ def get_client(self) -> "VoyageAIClient":
         """Creates a VoyageAI python client to embed elements."""
         from voyageai import Client as VoyageAIClient
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         client = VoyageAIClient(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             max_retries=self.max_retries,
             timeout=self.timeout_in_seconds,
         )
@@ -80,8 +81,9 @@ def get_async_client(self) -> "AsyncVoyageAIClient":
         """Creates a VoyageAI python client to embed elements."""
         from voyageai import AsyncClient as AsyncVoyageAIClient
 
+        api_key = self.api_key.get_secret_value() if self.api_key else None
         client = AsyncVoyageAIClient(
-            api_key=self.api_key.get_secret_value(),
+            api_key=api_key,
             max_retries=self.max_retries,
             timeout=self.timeout_in_seconds,
         )