Add builtin scenarios

Author: sjmonson

src/guidellm/__main__.py

@@ -9,7 +9,7 @@
 from guidellm.backend import BackendType
 from guidellm.benchmark import ProfileType
 from guidellm.benchmark.entrypoints import benchmark_with_scenario
-from guidellm.benchmark.scenario import GenerativeTextScenario
+from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios
 from guidellm.config import print_config
 from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
 from guidellm.scheduler import StrategyType
@@ -38,7 +38,7 @@ def cli():
             dir_okay=False,
             path_type=Path,  # type: ignore[type-var]
         ),
-        click.STRING
+        click.Choice(get_builtin_scenarios()),
     ),
     default=None,
     help=("TODO: A scenario or path to config"),
@@ -280,8 +280,7 @@ def benchmark(
         elif isinstance(scenario, Path):
             _scenario = GenerativeTextScenario.from_file(scenario, overrides)
         else:
-            # TODO: Add support for builtin scenarios
-            raise NotImplementedError
+            _scenario = GenerativeTextScenario.from_builtin(scenario, overrides)
     except ValidationError as e:
         errs = e.errors(include_url=False, include_context=True, include_input=True)
         param_name = "--" + str(errs[0]["loc"][0]).replace("_", "-")

src/guidellm/benchmark/scenario.py

@@ -1,6 +1,7 @@
 from collections.abc import Iterable
+from functools import cache
 from pathlib import Path
-from typing import Annotated, Any, Literal, Optional, Union
+from typing import Annotated, Any, Literal, Optional, TypeVar, Union
 
 from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
 from pydantic import BeforeValidator, Field, NonNegativeInt, PositiveFloat, PositiveInt
@@ -13,7 +14,14 @@
 from guidellm.objects.pydantic import StandardBaseModel
 from guidellm.scheduler.strategy import StrategyType
 
-__ALL__ = ["Scenario", "GenerativeTextScenario"]
+__ALL__ = ["Scenario", "GenerativeTextScenario", "get_builtin_scenarios"]
+
+SCENARIO_DIR = Path(__file__).parent / "scenarios/"
+
+
+@cache
+def get_builtin_scenarios() -> list[str]:
+    return [p.stem for p in SCENARIO_DIR.glob("*.json")]
 
 
 def parse_float_list(value: Union[str, float, list[float]]) -> list[float]:
@@ -32,9 +40,21 @@ def parse_float_list(value: Union[str, float, list[float]]) -> list[float]:
         ) from err
 
 
+T = TypeVar("T", bound="Scenario")
+
+
 class Scenario(StandardBaseModel):
     target: str
 
+    @classmethod
+    def from_builtin(cls: type[T], name: str, overrides: Optional[dict] = None) -> T:
+        filename = SCENARIO_DIR / f"{name}.json"
+
+        if not filename.is_file():
+            raise ValueError(f"{name} is not a vaild builtin scenario")
+
+        return cls.from_file(filename, overrides)
+
 
 class GenerativeTextScenario(Scenario):
     # FIXME: This solves an issue with Pydantic and class types