Merge branch 'main' into support_sharegpt

Author: sjmonson

src/guidellm/__main__.py

@@ -33,7 +33,7 @@
 try:
     import uvloop
 except ImportError:
-    uvloop = None # type: ignore[assignment] # Optional dependency
+    uvloop = None  # type: ignore[assignment] # Optional dependency
 
 from guidellm.backends import BackendType
 from guidellm.benchmark import (
@@ -116,6 +116,7 @@ def benchmark():
 )
 @click.option(
     "--scenario",
+    "-c",
     type=cli_tools.Union(
         click.Path(
             exists=True,
@@ -392,8 +393,10 @@ def run(**kwargs):
     disable_progress = kwargs.pop("disable_progress", False)
 
     try:
+        # Only set CLI args that differ from click defaults
+        new_kwargs = cli_tools.set_if_not_default(click.get_current_context(), **kwargs)
         args = BenchmarkGenerativeTextArgs.create(
-            scenario=kwargs.pop("scenario", None), **kwargs
+            scenario=new_kwargs.pop("scenario", None), **new_kwargs
         )
     except ValidationError as err:
         # Translate pydantic valdation error to click argument error

src/guidellm/benchmark/benchmarker.py

@@ -13,7 +13,7 @@
 import uuid
 from abc import ABC
 from collections.abc import AsyncIterator, Iterable
-from typing import Generic
+from typing import Any, Generic
 
 from guidellm.benchmark.profile import Profile
 from guidellm.benchmark.progress import BenchmarkerProgress

src/guidellm/benchmark/schemas.py

@@ -23,7 +23,17 @@
 from typing import Any, ClassVar, Literal, TypeVar, cast
 
 import yaml
-from pydantic import ConfigDict, Field, computed_field, model_serializer
+from pydantic import (
+    AliasChoices,
+    AliasGenerator,
+    ConfigDict,
+    Field,
+    ValidationError,
+    ValidatorFunctionWrapHandler,
+    computed_field,
+    field_validator,
+    model_serializer,
+)
 from torch.utils.data import Sampler
 from transformers import PreTrainedTokenizerBase
 
@@ -1142,7 +1152,8 @@ def update_estimate(
         )
         request_duration = (
             (request_end_time - request_start_time)
-            if request_end_time and request_start_time else None
+            if request_end_time and request_start_time
+            else None
         )
 
         # Always track concurrency
@@ -1669,7 +1680,7 @@ def compile(
         estimated_state: EstimatedBenchmarkState,
         scheduler_state: SchedulerState,
         profile: Profile,
-        requests: Iterable,
+        requests: Iterable,  # noqa: ARG003
         backend: BackendInterface,
         environment: Environment,
         strategy: SchedulingStrategy,
@@ -1787,9 +1798,8 @@ def create(
                 scenario_data = scenario_data["args"]
             constructor_kwargs.update(scenario_data)
 
-        for key, value in kwargs.items():
-            if value != cls.get_default(key):
-                constructor_kwargs[key] = value
+        # Apply overrides from kwargs
+        constructor_kwargs.update(kwargs)
 
         return cls.model_validate(constructor_kwargs)
 
@@ -1818,13 +1828,19 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
         else:
             return factory({})  # type: ignore[call-arg] # Confirmed correct at runtime by code above
 
-
-
     model_config = ConfigDict(
         extra="ignore",
         use_enum_values=True,
         from_attributes=True,
         arbitrary_types_allowed=True,
+        validate_by_alias=True,
+        validate_by_name=True,
+        alias_generator=AliasGenerator(
+            # Support field names with hyphens
+            validation_alias=lambda field_name: AliasChoices(
+                field_name, field_name.replace("_", "-")
+            ),
+        ),
     )
 
     # Required
@@ -1838,7 +1854,7 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
     profile: StrategyType | ProfileType | Profile = Field(
         default="sweep", description="Benchmark profile or scheduling strategy type"
     )
-    rate: float | list[float] | None = Field(
+    rate: list[float] | None = Field(
         default=None, description="Request rate(s) for rate-based scheduling"
     )
     # Backend configuration
@@ -1871,6 +1887,12 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
     data_request_formatter: DatasetPreprocessor | dict[str, str] | str = Field(
         default="chat_completions",
         description="Request formatting preprocessor or template name",
+        validation_alias=AliasChoices(
+            "data_request_formatter",
+            "data-request-formatter",
+            "request_type",
+            "request-type",
+        ),
     )
     data_collator: Callable | Literal["generative"] | None = Field(
         default="generative", description="Data collator for batch processing"
@@ -1931,6 +1953,26 @@ def get_default(cls: type[BenchmarkGenerativeTextArgs], field: str) -> Any:
         default=None, description="Maximum global error rate (0-1) before stopping"
     )
 
+    @field_validator("data", "data_args", "rate", mode="wrap")
+    @classmethod
+    def single_to_list(
+        cls, value: Any, handler: ValidatorFunctionWrapHandler
+    ) -> list[Any]:
+        """
+        Ensures field is always a list.
+
+        :param value: Input value for the 'data' field
+        :return: List of data sources
+        """
+        try:
+            return handler(value)
+        except ValidationError as err:
+            # If validation fails, try wrapping the value in a list
+            if err.errors()[0]["type"] == "list_type":
+                return handler([value])
+            else:
+                raise
+
     @model_serializer
     def serialize_model(self):
         """

src/guidellm/data/deserializers/deserializer.py

@@ -107,8 +107,10 @@ def _deserialize_with_registered_deserializers(
 
         if len(errors) > 0:
             err_msgs = ""
+
             def sort_key(item):
                 return (isinstance(item[1], DataNotSupportedError), item[0])
+
             for key, err in sorted(errors.items(), key=sort_key):
                 err_msgs += f"\n  - Deserializer '{key}': ({type(err).__name__}) {err}"
             raise ValueError(
@@ -141,4 +143,3 @@ def _deserialize_with_specified_deserializer(
             random_seed=random_seed,
             **data_kwargs,
         )
-

src/guidellm/data/deserializers/synthetic.py

@@ -9,7 +9,7 @@
 import yaml
 from datasets import Features, IterableDataset, Value
 from faker import Faker
-from pydantic import ConfigDict, Field, model_validator
+from pydantic import ConfigDict, Field, ValidationError, model_validator
 from transformers import PreTrainedTokenizerBase
 
 from guidellm.data.deserializers.deserializer import (
@@ -242,6 +242,10 @@ def __call__(
         if (config := self._load_config_str(data)) is not None:
             return self(config, processor_factory, random_seed, **data_kwargs)
 
+        # Try to parse dict-like data directly
+        if (config := self._load_config_dict(data)) is not None:
+            return self(config, processor_factory, random_seed, **data_kwargs)
+
         if not isinstance(data, SyntheticTextDatasetConfig):
             raise DataNotSupportedError(
                 "Unsupported data for SyntheticTextDatasetDeserializer, "
@@ -266,6 +270,15 @@ def __call__(
             ),
         )
 
+    def _load_config_dict(self, data: Any) -> SyntheticTextDatasetConfig | None:
+        if not isinstance(data, dict | list):
+            return None
+
+        try:
+            return SyntheticTextDatasetConfig.model_validate(data)
+        except ValidationError:
+            return None
+
     def _load_config_file(self, data: Any) -> SyntheticTextDatasetConfig | None:
         if (not isinstance(data, str) and not isinstance(data, Path)) or (
             not Path(data).is_file()

src/guidellm/data/loaders.py

@@ -17,7 +17,6 @@
 __all__ = ["DataLoader", "DatasetsIterator"]
 
 
-
 class DatasetsIterator(TorchIterableDataset):
     def __init__(
         self,

src/guidellm/data/preprocessors/formatters.py

@@ -56,9 +56,7 @@ def __init__(
         self.stream: bool = stream
         self.max_tokens: int | None = max_tokens or max_completion_tokens
 
-    def __call__(
-        self, columns: dict[str, list[Any]]
-    ) -> GenerationRequest:
+    def __call__(self, columns: dict[str, list[Any]]) -> GenerationRequest:
         """
         :param columns: A dict of GenerativeDatasetColumnType to Any
         """
@@ -396,9 +394,7 @@ def __call__(  # noqa: C901
 class GenerativeAudioTranslationRequestFormatter(
     GenerativeAudioTranscriptionRequestFormatter
 ):
-    def __call__(
-        self, columns: dict[str, list[Any]]
-    ) -> GenerationRequest:
+    def __call__(self, columns: dict[str, list[Any]]) -> GenerationRequest:
         result = super().__call__(columns)
         result.request_type = "audio_translations"
         return result

src/guidellm/data/preprocessors/mappers.py

@@ -167,9 +167,7 @@ def __init__(
             dict[GenerativeDatasetColumnType, list[tuple[int, str]]] | None
         )
 
-    def __call__(
-        self, row: dict[str, Any]
-    ) -> dict[str, list[Any]]:
+    def __call__(self, row: dict[str, Any]) -> dict[str, list[Any]]:
         if self.datasets_column_mappings is None:
             raise ValueError("DefaultGenerativeColumnMapper not setup with data.")
 

src/guidellm/data/preprocessors/preprocessor.py

@@ -12,8 +12,7 @@
 
 @runtime_checkable
 class DatasetPreprocessor(Protocol):
-    def __call__(self, item: dict[str, Any]) -> (
-        GenerationRequest | dict[str, Any]): ...
+    def __call__(self, item: dict[str, Any]) -> GenerationRequest | dict[str, Any]: ...
 
 
 @runtime_checkable

src/guidellm/preprocess/dataset.py

@@ -238,7 +238,7 @@ def process_dataset(
     prompt_tokens: str | Path,
     output_tokens: str | Path,
     processor_args: dict[str, Any] | None = None,
-    data_args: dict[str, Any] | None = None,
+    data_args: dict[str, Any] | None = None,  # noqa: ARG001
     short_prompt_strategy: ShortPromptStrategy = ShortPromptStrategy.IGNORE,
     pad_char: str | None = None,
     concat_delimiter: str | None = None,