fix: Support custom models in openai text embedder (#5525)

## Changes Made

Currently the openai text embedder asserts that the model must be an
openai model, however this does not work if user passes in a custom
BASE_URL that routes to an openai compatible server of a open source
model like qwen.

This PR elides the model check if user passes in BASE_URL, and also
allows user to pass in custom `embedding_dimensions`

## Related Issues

<!-- Link to related GitHub issues, e.g., "Closes #123" -->

## Checklist

- [ ] Documented in API Docs (if applicable)
- [ ] Documented in User Guide (if applicable)
- [ ] If adding a new documentation page, doc is added to
`docs/mkdocs.yml` navigation
- [ ] Documentation builds and is formatted properly

Author: colin-ho

daft/ai/openai/protocols/text_embedder.py

@@ -63,7 +63,7 @@ class OpenAITextEmbedderDescriptor(TextEmbedderDescriptor):
     model_options: Options
 
     def __post_init__(self) -> None:
-        if self.model_name not in _models:
+        if self.provider_options.get("base_url") is None and self.model_name not in _models:
             supported_models = ", ".join(_models.keys())
             raise ValueError(
                 f"Unsupported OpenAI embedding model '{self.model_name}', expected one of: {supported_models}"
@@ -79,10 +79,12 @@ def get_options(self) -> Options:
         return self.model_options
 
     def get_dimensions(self) -> EmbeddingDimensions:
+        if self.model_options.get("embedding_dimensions") is not None:
+            return EmbeddingDimensions(size=self.model_options["embedding_dimensions"], dtype=DataType.float32())
         return _models[self.model_name].dimensions
 
     def get_udf_options(self) -> UDFOptions:
-        return get_http_udf_options()
+        return UDFOptions(concurrency=None, num_gpus=None)
 
     def is_async(self) -> bool:
         return True