Add Detailed Benchmark Progress Tracking and Enhanced Report Generation Features (#28)

## Summary

This PR introduces enhancements to the benchmarking and report
generation functionalities in the GuideLLM project. Key updates include
the integration of detailed progress tracking using the rich library,
improvements to performance statistics, and adjustments to report output
formats.

## Details

- Added `BenchmarkReportProgress` class to manage real-time progress
updates for benchmarks and report generation using `rich`.
- Updated the report structure to include comprehensive performance
statistics, such as request latency, time to first token, and
inter-token latency.
- Introduced new methods to generate detailed tables summarizing
requests, data tokens, and performance metrics in reports.
- Enhanced the executor logic to handle different benchmark modes (e.g.,
sweep, synchronous, throughput) more effectively.
- Refined error handling and logging for better clarity and debugging.
- Modified unit tests to cover new functionalities and ensure robustness
across various scenarios.
- Adjusted configuration and settings to optimize benchmarking
capabilities and ensure consistent output.

## Test Plan

- Added new unit tests to verify the functionality of
`BenchmarkReportProgress` and other report generation methods.
- Performed end-to-end testing of different benchmark modes to ensure
accurate real-time progress tracking and report generation.
- Verified that all existing tests pass without any regression failures.

<img width="716" alt="Screen Shot 2024-08-22 at 4 21 29 AM"
src="https://github.com/user-attachments/assets/cf336607-3be2-47aa-b570-5bf7c44da1be">
<img width="1279" alt="Screen Shot 2024-08-22 at 4 59 21 AM"
src="https://github.com/user-attachments/assets/fe6a167b-1fa3-41ca-b95d-882ddd08bc1c">

Author: markurtz

.pre-commit-config.yaml

@@ -26,6 +26,7 @@ repos:
         pydantic_settings,
         pyyaml,
         requests,
+        rich,
         transformers,
 
         # dev dependencies

pyproject.toml

@@ -35,6 +35,7 @@ dependencies = [
     "pydantic-settings>=2.0.0",
     "pyyaml>=6.0.0",
     "requests",
+    "rich",
     "transformers",
 ]
 
@@ -182,6 +183,7 @@ select = [
     "N806", # allow uppercase variable names in tests
     "PGH003", # allow general ignores in tests
     "S106", # allow hardcoded passwords in tests
+    "PLR0915", # allow complext statements in tests
 ]
 
 [tool.ruff.lint.isort]

src/guidellm/__init__.py

@@ -3,7 +3,15 @@
 evaluating and benchmarking large language models (LLMs).
 """
 
+import os
+
+import transformers  # type: ignore  # noqa: PGH003
+
 from .config import settings
 from .logger import configure_logger, logger
 
 __all__ = ["configure_logger", "logger", "settings"]
+
+
+os.environ["TOKENIZERS_PARALLELISM"] = "false"  # Silence warnings for tokenizers
+transformers.logging.set_verbosity_error()  # Silence warnings for transformers

src/guidellm/backend/base.py

@@ -51,6 +51,12 @@ class Backend(ABC):
 
     :cvar _registry: A dictionary that maps BackendEngine types to backend classes.
     :type _registry: Dict[BackendEngine, Type[Backend]]
+    :param type_: The type of the backend.
+    :type type_: BackendEngine
+    :param target: The target URL for the backend.
+    :type target: str
+    :param model: The model used by the backend.
+    :type model: str
     """
 
     _registry: Dict[BackendEngine, "Type[Backend]"] = {}
@@ -96,6 +102,11 @@ def create(cls, backend_type: BackendEngine, **kwargs) -> "Backend":
 
         return Backend._registry[backend_type](**kwargs)
 
+    def __init__(self, type_: BackendEngine, target: str, model: str):
+        self._type = type_
+        self._target = target
+        self._model = model
+
     @property
     def default_model(self) -> str:
         """
@@ -107,6 +118,36 @@ def default_model(self) -> str:
         """
         return _cachable_default_model(self)
 
+    @property
+    def type_(self) -> BackendEngine:
+        """
+        Get the type of the backend.
+
+        :return: The type of the backend.
+        :rtype: BackendEngine
+        """
+        return self._type
+
+    @property
+    def target(self) -> str:
+        """
+        Get the target URL for the backend.
+
+        :return: The target URL.
+        :rtype: str
+        """
+        return self._target
+
+    @property
+    def model(self) -> str:
+        """
+        Get the model used by the backend.
+
+        :return: The model name.
+        :rtype: str
+        """
+        return self._model
+
     async def submit(self, request: TextGenerationRequest) -> TextGenerationResult:
         """
         Submit a text generation request and return the result.

src/guidellm/backend/openai.py

@@ -45,10 +45,19 @@ def __init__(
         self._request_args: Dict = request_args
         api_key: str = openai_api_key or settings.openai.api_key
 
+        if not api_key:
+            err = ValueError(
+                "`GUIDELLM__OPENAI__API_KEY` environment variable or "
+                "--openai-api-key CLI parameter must be specified for the "
+                "OpenAI backend."
+            )
+            logger.error("{}", err)
+            raise err
+
         if target:
             base_url = target
         elif host and port:
-            base_url = f"{host}:{port}"
+            base_url = f"{host}:{port}/v1"
         elif settings.openai.base_url:
             base_url = settings.openai.base_url
         else:
@@ -61,22 +70,11 @@ def __init__(
 
         self._async_client = AsyncOpenAI(api_key=api_key, base_url=base_url)
         self._client = OpenAI(api_key=api_key, base_url=base_url)
-
-        self.validate_connection()
         self._model = model or self.default_model
 
+        super().__init__(type_="openai_server", target=base_url, model=self._model)
         logger.info("OpenAI {} Backend listening on {}", self._model, base_url)
 
-    @property
-    def model(self) -> str:
-        """
-        Get the model used by this backend.
-
-        :return: The model name.
-        :rtype: str
-        """
-        return self._model
-
     async def make_request(
         self,
         request: TextGenerationRequest,

src/guidellm/config.py

@@ -43,7 +43,7 @@ class LoggingSettings(BaseModel):
 
     disabled: bool = False
     clear_loggers: bool = True
-    console_log_level: str = "INFO"
+    console_log_level: str = "WARNING"
     log_file: Optional[str] = None
     log_file_level: Optional[str] = None
 
@@ -98,7 +98,7 @@ class OpenAISettings(BaseModel):
     """
 
     # OpenAI API key.
-    api_key: str = ""
+    api_key: str = "invalid_token"
 
     # OpenAI-compatible server URL
     # NOTE: The default value is default address of llama.cpp web server
@@ -141,8 +141,8 @@ class Settings(BaseSettings):
     # general settings
     env: Environment = Environment.PROD
     request_timeout: int = 30
-    max_concurrency: int = 128
-    num_sweep_profiles: int = 10
+    max_concurrency: int = 512
+    num_sweep_profiles: int = 9
     logging: LoggingSettings = LoggingSettings()
 
     # Data settings