Merge branch 'main' into switch-itl-to-tpot

Author: DaltheCow

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
   hooks:
     - id: pdm-lock-check
       name: check lock file matches pyproject
+      entry: sh -c "pdm lock -v --check || pdm lock --update-reuse"
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.11.7
   hooks:

src/guidellm/benchmark/aggregator.py

@@ -32,11 +32,11 @@
     GenerationRequest,
     GenerativeRequestLoaderDescription,
     RequestLoaderDescription,
+    RequestT,
+    ResponseT,
 )
 from guidellm.scheduler import (
     GenerativeRequestsWorkerDescription,
-    RequestT,
-    ResponseT,
     SchedulerRequestResult,
     WorkerDescription,
 )

src/guidellm/benchmark/benchmarker.py

@@ -27,12 +27,12 @@
     GenerationRequest,
     GenerativeRequestLoaderDescription,
     RequestLoaderDescription,
+    RequestT,
+    ResponseT,
 )
 from guidellm.scheduler import (
     GenerativeRequestsWorker,
     RequestsWorker,
-    RequestT,
-    ResponseT,
     Scheduler,
     SchedulerRequestResult,
     SchedulingStrategy,

src/guidellm/config.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections.abc import Sequence
 from enum import Enum
 from typing import Literal, Optional
@@ -131,8 +132,12 @@ class Settings(BaseSettings):
 
     # Scheduler settings
     max_concurrency: int = 512
-    max_worker_processes: int = 10
-    max_add_requests_per_loop: int = 20
+    max_worker_processes: int = Field(
+        # use number of CPUs - 1, but at least 10
+        default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
+    )
+    min_queued_requests: int = 20
+    scheduler_start_delay: float = 5
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()

src/guidellm/logger.py

@@ -71,7 +71,8 @@ def configure_logger(config: LoggingSettings = settings.logging):
     logger.add(
         sys.stdout,
         level=config.console_log_level.upper(),
-        format="{time} | {function} | {level} - {message}",
+        format="<green>{time:YY-MM-DD HH:mm:ss}</green>|<level>{level: <8}</level> \
+        |<cyan>{name}:{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>"
     )
 
     if config.log_file or config.log_file_level:

src/guidellm/request/__init__.py

@@ -5,11 +5,14 @@
     RequestLoaderDescription,
 )
 from .request import GenerationRequest
+from .types import RequestT, ResponseT
 
 __all__ = [
     "GenerationRequest",
     "GenerativeRequestLoader",
     "GenerativeRequestLoaderDescription",
     "RequestLoader",
     "RequestLoaderDescription",
+    "RequestT",
+    "ResponseT",
 ]

src/guidellm/request/loader.py

@@ -30,10 +30,10 @@ class RequestLoaderDescription(StandardBaseModel):
 
 class RequestLoader(Iterable):
     @abstractmethod
-    def __iter__(self): ...
+    def __iter__(self) -> Iterator: ...
 
     @abstractmethod
-    def __len__(self): ...
+    def __len__(self) -> int: ...
 
     @property
     @abstractmethod

src/guidellm/scheduler/types.py renamed to src/guidellm/request/types.py

@@ -1,6 +1,9 @@
 from typing import TypeVar
 
-__all__ = ["RequestT", "ResponseT"]
+__all__ = [
+    "RequestT",
+    "ResponseT",
+]
 
 
 RequestT = TypeVar("RequestT")

src/guidellm/scheduler/__init__.py

@@ -15,14 +15,12 @@
     ThroughputStrategy,
     strategy_display_str,
 )
-from .types import RequestT, ResponseT
 from .worker import (
     GenerativeRequestsWorker,
     GenerativeRequestsWorkerDescription,
     RequestsWorker,
     ResolveStatus,
     WorkerDescription,
-    WorkerProcessRequest,
     WorkerProcessResult,
 )
 
@@ -32,10 +30,8 @@
     "ConcurrentStrategy",
     "GenerativeRequestsWorker",
     "GenerativeRequestsWorkerDescription",
-    "RequestT",
     "RequestsWorker",
     "ResolveStatus",
-    "ResponseT",
     "Scheduler",
     "SchedulerRequestInfo",
     "SchedulerRequestResult",
@@ -46,7 +42,6 @@
     "SynchronousStrategy",
     "ThroughputStrategy",
     "WorkerDescription",
-    "WorkerProcessRequest",
     "WorkerProcessResult",
     "strategy_display_str",
 ]

src/guidellm/scheduler/queues.py

@@ -0,0 +1,25 @@
+"""
+Helper module for importing the correct queue types.
+"""
+
+from dataclasses import dataclass
+from queue import Empty as QueueEmpty
+from queue import Full as QueueFull
+from queue import Queue
+from typing import Generic
+
+from guidellm.request.types import RequestT, ResponseT
+from guidellm.scheduler.result import WorkerProcessRequest, WorkerProcessResult
+
+__all__ = [
+    "MPQueues",
+    "Queue",
+    "QueueEmpty",
+    "QueueFull",
+]
+
+
+@dataclass
+class MPQueues(Generic[RequestT, ResponseT]):
+    requests: Queue[WorkerProcessRequest[RequestT, ResponseT]]
+    responses: Queue[WorkerProcessResult[RequestT, ResponseT]]