add descriptions to all embedder config pydantic model fields (#478)

Author: rbiseck3

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.16
+
+* **Add embedder config field descriptions**
+
 ## 1.0.15
 
 ### Fixes

requirements/base.txt

@@ -1,7 +1,6 @@
 python-dateutil
 # Pydantic generic Secret only introduced in 2.7
 pydantic>=2.7
-dataclasses_json
 tqdm
 click
 opentelemetry-sdk

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.0.15"  # pragma: no cover
+__version__ = "1.0.16"  # pragma: no cover

unstructured_ingest/embed/azure_openai.py

@@ -16,8 +16,10 @@
 
 class AzureOpenAIEmbeddingConfig(OpenAIEmbeddingConfig):
     api_version: str = Field(description="Azure API version", default="2024-06-01")
-    azure_endpoint: str
-    embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
+    azure_endpoint: str = Field(description="Azure endpoint")
+    embedder_model_name: str = Field(
+        default="text-embedding-ada-002", alias="model_name", description="Azure OpenAI model name"
+    )
 
     @requires_dependencies(["openai"], extras="openai")
     def get_client(self) -> "AzureOpenAI":

unstructured_ingest/embed/bedrock.py

@@ -58,10 +58,14 @@ def conform_query(query: str, provider: str) -> dict:
 
 
 class BedrockEmbeddingConfig(EmbeddingConfig):
-    aws_access_key_id: SecretStr
-    aws_secret_access_key: SecretStr
-    region_name: str = "us-west-2"
-    embedder_model_name: str = Field(default="amazon.titan-embed-text-v1", alias="model_name")
+    aws_access_key_id: SecretStr = Field(description="aws access key id")
+    aws_secret_access_key: SecretStr = Field(description="aws secret access key")
+    region_name: str = Field(description="aws region name", default="us-west-2")
+    embedder_model_name: str = Field(
+        default="amazon.titan-embed-text-v1",
+        alias="model_name",
+        description="AWS Bedrock model name",
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):

unstructured_ingest/embed/huggingface.py

@@ -15,11 +15,18 @@
 
 
 class HuggingFaceEmbeddingConfig(EmbeddingConfig):
-    embedder_model_name: Optional[str] = Field(default="all-MiniLM-L6-v2", alias="model_name")
+    embedder_model_name: Optional[str] = Field(
+        default="all-MiniLM-L6-v2", alias="model_name", description="HuggingFace model name"
+    )
     embedder_model_kwargs: Optional[dict] = Field(
-        default_factory=lambda: {"device": "cpu"}, alias="model_kwargs"
+        default_factory=lambda: {"device": "cpu"},
+        alias="model_kwargs",
+        description="additional model parameters",
+    )
+    encode_kwargs: Optional[dict] = Field(
+        default_factory=lambda: {"normalize_embeddings": False},
+        description="additional embedding parameters",
     )
-    encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
 
     @requires_dependencies(
         ["sentence_transformers"],

unstructured_ingest/embed/mixedbreadai.py

@@ -34,10 +34,13 @@ class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
 
     api_key: SecretStr = Field(
         default_factory=lambda: SecretStr(os.environ.get("MXBAI_API_KEY")),
+        description="API key for Mixedbread AI",
     )
 
     embedder_model_name: str = Field(
-        default="mixedbread-ai/mxbai-embed-large-v1", alias="model_name"
+        default="mixedbread-ai/mxbai-embed-large-v1",
+        alias="model_name",
+        description="Mixedbread AI model name",
     )
 
     @requires_dependencies(

unstructured_ingest/embed/octoai.py

@@ -25,8 +25,12 @@
 
 class OctoAiEmbeddingConfig(EmbeddingConfig):
     api_key: SecretStr
-    embedder_model_name: str = Field(default="thenlper/gte-large", alias="model_name")
-    base_url: str = Field(default="https://text.octoai.run/v1")
+    embedder_model_name: str = Field(
+        default="thenlper/gte-large", alias="model_name", description="octoai model name"
+    )
+    base_url: str = Field(
+        default="https://text.octoai.run/v1", description="optional override for the base url"
+    )
 
     def wrap_error(self, e: Exception) -> Exception:
         if is_internal_error(e=e):

unstructured_ingest/embed/openai.py

@@ -24,9 +24,11 @@
 
 
 class OpenAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
-    embedder_model_name: str = Field(default="text-embedding-ada-002", alias="model_name")
-    base_url: Optional[str] = None
+    api_key: SecretStr = Field(description="API key for OpenAI")
+    embedder_model_name: str = Field(
+        default="text-embedding-ada-002", alias="model_name", description="OpenAI model name"
+    )
+    base_url: Optional[str] = Field(default=None, description="optional override for the base url")
 
     @requires_dependencies(["openai"], extras="openai")
     def wrap_error(self, e: Exception) -> Exception:

unstructured_ingest/embed/togetherai.py

@@ -20,9 +20,11 @@
 
 
 class TogetherAIEmbeddingConfig(EmbeddingConfig):
-    api_key: SecretStr
+    api_key: SecretStr = Field(description="API key for Together AI")
     embedder_model_name: str = Field(
-        default="togethercomputer/m2-bert-80M-8k-retrieval", alias="model_name"
+        default="togethercomputer/m2-bert-80M-8k-retrieval",
+        alias="model_name",
+        description="Together AI model name",
     )
 
     def wrap_error(self, e: Exception) -> Exception: