latest working state with data loaders and benchmarking API

Author: markurtz

plot.png

@@ -32,6 +32,7 @@ dependencies = [
     "loguru",
     "numpy",
     "pillow",
+    "protobuf",
     "pydantic>=2.0.0",
     "pydantic-settings>=2.0.0",
     "pyyaml>=6.0.0",

pyproject.toml

@@ -14,6 +14,6 @@
 
 from .config import settings
 from .logger import configure_logger, logger
-from .main import generate_benchmark_report
+# from .main import generate_benchmark_report
 
 __all__ = ["configure_logger", "logger", "settings", "generate_benchmark_report"]

src/guidellm/__init__.py

@@ -1,4 +1,3 @@
-import asyncio
 from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import Any, AsyncGenerator, Dict, List, Literal, Optional, Type, Union
@@ -110,7 +109,7 @@ def info(self) -> Dict[str, Any]:
         """
         ...
 
-    def validate(self):
+    async def validate(self):
         """
         Handle final setup and validate the backend is ready for use.
         If not successful, raises the appropriate exception.
@@ -121,13 +120,10 @@ def validate(self):
         if not models:
             raise ValueError("No models available for the backend")
 
-        async def _test_request():
-            async for _ in self.text_completions(
-                prompt="Test connection", output_token_count=1
-            ):  # type: ignore[attr-defined]
-                pass
-
-        asyncio.run(_test_request())
+        async for _ in self.text_completions(
+            prompt="Test connection", output_token_count=1
+        ):  # type: ignore[attr-defined]
+            pass
 
     @abstractmethod
     def check_setup(self):

src/guidellm/backend/backend.py

@@ -202,7 +202,7 @@ async def text_completions(  # type: ignore[override]
 
         try:
             async for resp in self._iterative_completions_request(
-                type_="text",
+                type_="text_completions",
                 request_id=request_id,
                 request_prompt_tokens=prompt_token_count,
                 request_output_tokens=output_token_count,
@@ -277,7 +277,7 @@ async def chat_completions(  # type: ignore[override]
 
         try:
             async for resp in self._iterative_completions_request(
-                type_="chat",
+                type_="chat_completions",
                 request_id=request_id,
                 request_prompt_tokens=prompt_token_count,
                 request_output_tokens=output_token_count,
@@ -403,16 +403,16 @@ def _create_chat_messages(
 
     async def _iterative_completions_request(
         self,
-        type_: Literal["text", "chat"],
+        type_: Literal["text_completions", "chat_completions"],
         request_id: Optional[str],
         request_prompt_tokens: Optional[int],
         request_output_tokens: Optional[int],
         headers: Dict,
         payload: Dict,
     ) -> AsyncGenerator[Union[StreamingTextResponse, ResponseSummary], None]:
-        if type_ == "text":
+        if type_ == "text_completions":
             target = f"{self.target}{TEXT_COMPLETIONS_PATH}"
-        elif type_ == "chat":
+        elif type_ == "chat_completions":
             target = f"{self.target}{CHAT_COMPLETIONS_PATH}"
         else:
             raise ValueError(f"Unsupported type: {type_}")
@@ -525,15 +525,15 @@ async def _iterative_completions_request(
 
     @staticmethod
     def _extract_completions_delta_content(
-        type_: Literal["text", "chat"], data: Dict
+        type_: Literal["text_completions", "chat_completions"], data: Dict
     ) -> Optional[str]:
         if "choices" not in data or not data["choices"]:
             return None
 
-        if type_ == "text":
+        if type_ == "text_completions":
             return data["choices"][0]["text"]
 
-        if type_ == "chat":
+        if type_ == "chat_completions":
             return data["choices"][0]["delta"]["content"]
 
         raise ValueError(f"Unsupported type: {type_}")