Allow reexporting reimported benchmarks

Author: jaredoconnell

pyproject.toml

@@ -56,6 +56,7 @@ dependencies = [
     "pyyaml>=6.0.0",
     "rich",
     "transformers",
+    "click-default-group~=1.2.4"
 ]
 
 [project.optional-dependencies]

src/guidellm/__main__.py

@@ -3,14 +3,15 @@
 import json
 from pathlib import Path
 from typing import get_args
+from click_default_group import DefaultGroup
 
 import click
 
 from guidellm.backend import BackendType
 from guidellm.benchmark import (
     ProfileType,
     benchmark_generative_text,
-    display_benchmarks_report,
+    reimport_benchmarks_report,
 )
 from guidellm.config import print_config
 from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset
@@ -48,8 +49,18 @@ def parse_number_str(ctx, param, value):  # noqa: ARG001
 def cli():
     pass
 
+@cli.group(
+    help="Commands to run a new benchmark or load a prior one.",
+    cls=DefaultGroup,
+    default="run",
+    default_if_no_args=True,
+)
+def benchmark():
+    pass
 
-@cli.command(
+
+@benchmark.command(
+    "run",
     help="Run a benchmark against a generative model using the specified arguments."
 )
 @click.option(
@@ -234,7 +245,7 @@ def cli():
     type=int,
     help="The random seed to use for benchmarking to ensure reproducibility.",
 )
-def benchmark(
+def run(
     target,
     backend_type,
     backend_args,
@@ -286,14 +297,32 @@ def benchmark(
     )
 
 
-@cli.command(help="Redisplay a saved benchmark report.")
+@benchmark.command(help="Load a saved benchmark report.")
 @click.argument(
     "path",
     type=click.Path(file_okay=True, dir_okay=False, exists=True),
     default=Path.cwd() / "benchmarks.json",
 )
-def display(path):
-    display_benchmarks_report(path)
+@click.option(
+    "--output-path",
+    type=click.Path(file_okay=True, dir_okay=True, exists=False),
+    default=None,
+    is_flag=False,
+    flag_value=Path.cwd() / "benchmarks_reexported.json",
+    help=(
+        "Allows re-exporting the benchmarks to another format."
+        "The path to save the output to. If it is a directory, "
+        "it will save benchmarks.json under it. "
+        "Otherwise, json, yaml, or csv files are supported for output types "
+        "which will be read from the extension for the file path."
+        "Optional. If the output path flag is not provided, the benchmarks "
+        "will not be reexported. If the flag is present but no value is "
+        "specified, it will default to the current directory with the file "
+        "name benchmarks_reexported.json."
+    ),
+)
+def from_file(path, output_path):
+    reimport_benchmarks_report(path, output_path)
 
 
 def decode_escaped_str(_ctx, _param, value):
@@ -311,6 +340,7 @@ def decode_escaped_str(_ctx, _param, value):
 
 
 @cli.command(
+    short_help="Prints environment variable settings.",
     help=(
         "Print out the available configuration settings that can be set "
         "through environment variables."

src/guidellm/benchmark/__init__.py

@@ -12,7 +12,7 @@
     StatusBreakdown,
 )
 from .benchmarker import Benchmarker, BenchmarkerResult, GenerativeBenchmarker
-from .entrypoints import benchmark_generative_text, display_benchmarks_report
+from .entrypoints import benchmark_generative_text, reimport_benchmarks_report
 from .output import GenerativeBenchmarksConsole, GenerativeBenchmarksReport
 from .profile import (
     AsyncProfile,
@@ -63,5 +63,5 @@
     "ThroughputProfile",
     "benchmark_generative_text",
     "create_profile",
-    "display_benchmarks_report",
+    "reimport_benchmarks_report",
 ]

src/guidellm/benchmark/entrypoints.py

@@ -135,12 +135,18 @@ async def benchmark_generative_text(
 
     return report, saved_path
 
-def display_benchmarks_report(file: Path):
+def reimport_benchmarks_report(file: Path, output_path: Optional[Path]) -> None:
     """
-    The command-line entry point for displaying a benchmarks report.
+    The command-line entry point for re-importing and displaying an
+    existing benchmarks report. Can also specify
     Assumes the file provided exists.
     """
     console = GenerativeBenchmarksConsole(enabled=True)
     report = GenerativeBenchmarksReport.load_file(file)
     console.benchmarks = report.benchmarks
     console.print_full_report()
+
+    if output_path:
+        console.print_line("\nSaving benchmarks report...")
+        saved_path = report.save_file(output_path)
+        console.print_line(f"Benchmarks report saved to {saved_path}")

src/guidellm/benchmark/output.py

@@ -242,7 +242,7 @@ def _file_setup(
         if path_suffix in [".csv"]:
             return path, "csv"
 
-        raise ValueError(f"Unsupported file extension: {path_suffix} for {path}.")
+        raise ValueError(f"Unsupported file extension: {path_suffix} for {path}; expected json, yaml, or csv.")
 
     @staticmethod
     def _benchmark_desc_headers_and_values(

tests/unit/entrypoints/test_display_entrypoint.py renamed to tests/unit/entrypoints/test_benchmark_from_file_entrypoint.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-from guidellm.benchmark import display_benchmarks_report
+from guidellm.benchmark import reimport_benchmarks_report
 
 # Set to true to re-write the expected output.
 REGENERATE_ARTIFACTS = False
@@ -37,7 +37,7 @@ def test_display_entrypoint_yaml(capfd, get_test_asset_dir):
 def generic_test_display_entrypoint(filename, capfd, get_test_asset_dir):
     os.environ["COLUMNS"] = "180"  # CLI output depends on terminal width.
     asset_dir = get_test_asset_dir()
-    display_benchmarks_report(asset_dir / filename)
+    reimport_benchmarks_report(asset_dir / filename)
     out, err = capfd.readouterr()
     expected_output_path = asset_dir / "benchmarks_stripped_output.txt"
     if REGENERATE_ARTIFACTS: