feat: batch embeddings

Author: exiaohu

volcenginesdkarkruntime/resources/batch/_utils.py

@@ -0,0 +1,129 @@
+import asyncio
+import logging
+import time
+from datetime import datetime, timedelta
+from random import random
+from typing import Any, Awaitable, Callable, Optional, TypeVar
+
+import httpx
+
+from ..._constants import INITIAL_RETRY_DELAY, MAX_RETRY_DELAY
+from ..._exceptions import ArkAPIConnectionError, ArkAPIStatusError, ArkAPITimeoutError
+from ..._utils._model_breaker import ModelBreaker
+
+log: logging.Logger = logging.getLogger(__name__)
+
+
+def _calculate_retry_timeout(retry_times: int) -> float:
+    nbRetries = min(retry_times, MAX_RETRY_DELAY / INITIAL_RETRY_DELAY)
+    sleep_seconds = min(INITIAL_RETRY_DELAY * pow(2, nbRetries), MAX_RETRY_DELAY)
+    # Apply some jitter, plus-or-minus half a second.
+    jitter = 1 - 0.25 * random()
+    timeout = sleep_seconds * jitter
+    return timeout if timeout >= 0 else 0
+
+
+def _get_retry_after(response: httpx.Response) -> int | None:
+    retry_after = response.headers.get("Retry-After")
+    if retry_after is not None:
+        if retry_after.isdigit():
+            return int(retry_after)
+    return None
+
+
+def _should_retry(response: httpx.Response) -> bool:
+    # Retry on request timeouts.
+    if response.status_code == 408:
+        return True
+
+    # Retry on lock timeouts.
+    if response.status_code == 409:
+        return True
+
+    # Retry on rate limits.
+    if response.status_code == 429:
+        return True
+
+    # Retry internal errors.
+    if response.status_code >= 500:
+        return True
+
+    return False
+
+
+def get_request_last_time(
+    client: httpx.Client, timeout: Optional[Any] = None
+) -> datetime:
+    if timeout is None:
+        timeout = client.timeout
+    timeoutSeconds = 0
+    if isinstance(timeout, httpx.Timeout):
+        timeoutSeconds = timeout.read
+    elif isinstance(timeout, float):
+        timeoutSeconds = timeout
+    elif isinstance(timeout, int):
+        timeoutSeconds = timeout
+    else:
+        raise TypeError("timeout type {} is not supported".format(type(timeout)))
+    return datetime.now() + timedelta(seconds=timeoutSeconds)
+
+
+R = TypeVar("R")
+
+
+def with_batch_retry(
+    deadline: datetime,
+    breaker: ModelBreaker,
+    func: Callable[..., R],
+    *args,
+    **kwargs,
+) -> R:
+    retry_times = 0
+    while True:
+        breaker.wait()
+        if datetime.now() > deadline:
+            raise ArkAPITimeoutError(None, None)
+        try:
+            return func(*args, **kwargs)
+        except ArkAPIConnectionError:
+            waitTime = _calculate_retry_timeout(retry_times)
+            if datetime.now() + timedelta(seconds=waitTime) > deadline:
+                raise ArkAPITimeoutError(None, None)
+            time.sleep(waitTime)
+        except ArkAPIStatusError as err:
+            retry_after = _get_retry_after(err.response)
+            if retry_after is not None and retry_after > 0:
+                breaker.reset(retry_after)
+            if not _should_retry(err.response):
+                raise err
+
+        retry_times = retry_times + 1
+
+
+async def async_with_batch_retry(
+    deadline: datetime,
+    breaker: ModelBreaker,
+    func: Callable[..., Awaitable[R]],
+    *args,
+    **kwargs,
+) -> R:
+    retry_times = 0
+    while True:
+        await breaker.asyncwait()
+        if datetime.now() > deadline:
+            raise ArkAPITimeoutError(None, None)
+        try:
+            return await func(*args, **kwargs)
+        except ArkAPIConnectionError:
+            waitTime = _calculate_retry_timeout(retry_times)
+            if datetime.now() + timedelta(seconds=waitTime) > deadline:
+                raise ArkAPITimeoutError(None, None)
+            await asyncio.sleep(waitTime)
+        except ArkAPIStatusError as err:
+            retry_after = _get_retry_after(err.response)
+            if retry_after is not None and retry_after > 0:
+                breaker.reset(retry_after)
+            if not _should_retry(err.response):
+                raise err
+
+        retry_times = retry_times + 1

volcenginesdkarkruntime/resources/batch/batch.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 from ..._compat import cached_property
-from .chat.chat import Chat, AsyncChat
-from ..._resource import SyncAPIResource, AsyncAPIResource
+from ..._resource import AsyncAPIResource, SyncAPIResource
+from .chat.chat import AsyncChat, Chat
+from .embeddings import AsyncEmbeddings, Embeddings
 
 __all__ = ["Batch", "AsyncBatch"]
 
@@ -12,8 +13,16 @@ class Batch(SyncAPIResource):
     def chat(self) -> Chat:
         return Chat(self._client)
 
+    @cached_property
+    def embeddings(self) -> Embeddings:
+        return Embeddings(self._client)
+
 
 class AsyncBatch(AsyncAPIResource):
     @cached_property
     def chat(self) -> AsyncChat:
         return AsyncChat(self._client)
+
+    @cached_property
+    def embeddings(self) -> AsyncEmbeddings:
+        return AsyncEmbeddings(self._client)

volcenginesdkarkruntime/resources/batch/chat/chat.py

@@ -1,8 +1,8 @@
 from __future__ import annotations
 
 from ...._compat import cached_property
-from .completions import Completions, AsyncCompletions
-from ...._resource import SyncAPIResource, AsyncAPIResource
+from ...._resource import AsyncAPIResource, SyncAPIResource
+from .completions import AsyncCompletions, Completions
 
 __all__ = ["Chat", "AsyncChat"]