Merge branch 'main' into example1

Author: tukwila

.github/workflows/main.yml

@@ -217,7 +217,7 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./ui/out
+          publish_dir: ./src/ui/out
           destination_dir: ui/dev
           keep_files: false
           user_name: ${{ github.actor }}

.github/workflows/nightly.yml

@@ -238,7 +238,7 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./ui/out
+          publish_dir: ./src/ui/out
           destination_dir: ui/nightly
           keep_files: false
           user_name: ${{ github.actor }}

.github/workflows/release-candidate.yml

@@ -282,7 +282,7 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./ui/out
+          publish_dir: ./src/ui/out
           destination_dir: ui/release/latest
           keep_files: false
           user_name: ${{ github.actor }}

.github/workflows/release.yml

@@ -281,7 +281,7 @@ jobs:
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./ui/out
+          publish_dir: ./src/ui/out
           destination_dir: ui/latest
           keep_files: false
           user_name: ${{ github.actor }}

.pre-commit-config.yaml

@@ -11,6 +11,7 @@ repos:
   hooks:
     - id: pdm-lock-check
       name: check lock file matches pyproject
+      entry: sh -c "pdm lock -v --check || pdm lock --update-reuse"
 - repo: https://github.com/astral-sh/ruff-pre-commit
   rev: v0.11.7
   hooks:

src/guidellm/benchmark/aggregator.py

@@ -32,11 +32,11 @@
     GenerationRequest,
     GenerativeRequestLoaderDescription,
     RequestLoaderDescription,
+    RequestT,
+    ResponseT,
 )
 from guidellm.scheduler import (
     GenerativeRequestsWorkerDescription,
-    RequestT,
-    ResponseT,
     SchedulerRequestResult,
     WorkerDescription,
 )

src/guidellm/benchmark/benchmarker.py

@@ -27,12 +27,12 @@
     GenerationRequest,
     GenerativeRequestLoaderDescription,
     RequestLoaderDescription,
+    RequestT,
+    ResponseT,
 )
 from guidellm.scheduler import (
     GenerativeRequestsWorker,
     RequestsWorker,
-    RequestT,
-    ResponseT,
     Scheduler,
     SchedulerRequestResult,
     SchedulingStrategy,

src/guidellm/config.py

@@ -1,4 +1,5 @@
 import json
+import os
 from collections.abc import Sequence
 from enum import Enum
 from typing import Literal, Optional
@@ -131,8 +132,12 @@ class Settings(BaseSettings):
 
     # Scheduler settings
     max_concurrency: int = 512
-    max_worker_processes: int = 10
-    max_add_requests_per_loop: int = 20
+    max_worker_processes: int = Field(
+        # use number of CPUs - 1, but at least 10
+        default_factory=lambda: max((os.cpu_count() or 1) - 1, 10)
+    )
+    min_queued_requests: int = 20
+    scheduler_start_delay: float = 5
 
     # Data settings
     dataset: DatasetSettings = DatasetSettings()

src/guidellm/request/__init__.py

@@ -5,11 +5,14 @@
     RequestLoaderDescription,
 )
 from .request import GenerationRequest
+from .types import RequestT, ResponseT
 
 __all__ = [
     "GenerationRequest",
     "GenerativeRequestLoader",
     "GenerativeRequestLoaderDescription",
     "RequestLoader",
     "RequestLoaderDescription",
+    "RequestT",
+    "ResponseT",
 ]

src/guidellm/request/loader.py

@@ -30,10 +30,10 @@ class RequestLoaderDescription(StandardBaseModel):
 
 class RequestLoader(Iterable):
     @abstractmethod
-    def __iter__(self): ...
+    def __iter__(self) -> Iterator: ...
 
     @abstractmethod
-    def __len__(self): ...
+    def __len__(self) -> int: ...
 
     @property
     @abstractmethod