Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 7 additions & 6 deletions src/guidellm/benchmark/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import math
from abc import ABC, abstractmethod
from collections import OrderedDict
from copy import deepcopy
from datetime import datetime
from pathlib import Path
from typing import Any, ClassVar
Expand Down Expand Up @@ -36,6 +37,8 @@
safe_format_timestamp,
split_text_list_by_length,
)
from guidellm.utils.dict import recursive_key_update
from guidellm.utils.text import camelize_str

__all__ = [
"GenerativeBenchmarkerCSV",
Expand Down Expand Up @@ -711,22 +714,20 @@ async def finalize(self, report: GenerativeBenchmarksReport) -> Path:
:param report: The completed benchmark report.
:return: Path to the saved HTML file.
"""
import humps

output_path = self.output_path
if output_path.is_dir():
output_path = output_path / GenerativeBenchmarkerHTML.DEFAULT_FILE
output_path.parent.mkdir(parents=True, exist_ok=True)

data_builder = UIDataBuilder(report.benchmarks)
data = data_builder.to_dict()
camel_data = humps.camelize(data)
camel_data = recursive_key_update(deepcopy(data), camelize_str)

ui_api_data = {}
for key, value in camel_data.items():
placeholder_key = f"window.{humps.decamelize(key)} = {{}};"
for k, v in camel_data.items():
placeholder_key = f"window.{k} = {{}};"
replacement_value = (
f"window.{humps.decamelize(key)} = {json.dumps(value, indent=2)};\n"
f"window.{k} = {json.dumps(v, indent=2)};\n"
)
ui_api_data[placeholder_key] = replacement_value

Expand Down
12 changes: 6 additions & 6 deletions src/guidellm/presentation/data_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ class RunInfo(BaseModel):

@classmethod
def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]):
model = benchmarks[0].worker.backend_model or "N/A"
model = benchmarks[0].benchmarker.backend.get("model", "N/A")
timestamp = max(
bm.run_stats.start_time for bm in benchmarks if bm.start_time is not None
)
Expand Down Expand Up @@ -108,8 +108,8 @@ class WorkloadDetails(BaseModel):

@classmethod
def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]):
target = benchmarks[0].worker.backend_target
rate_type = benchmarks[0].args.profile.type_
target = benchmarks[0].benchmarker.backend.get("target", "N/A")
rate_type = benchmarks[0].scheduler.strategy.type_
successful_requests = [
req for bm in benchmarks for req in bm.requests.successful
]
Expand Down Expand Up @@ -152,13 +152,13 @@ def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]):
statistics=output_token_stats, buckets=output_token_buckets, bucket_width=1
)

min_start_time = benchmarks[0].run_stats.start_time
min_start_time = benchmarks[0].start_time

all_req_times = [
req.start_time - min_start_time
req.scheduler_info.started_at - min_start_time
for bm in benchmarks
for req in bm.requests.successful
if req.start_time is not None
if req.scheduler_info.started_at is not None
]
number_of_buckets = len(benchmarks)
request_over_time_buckets, bucket_width = Bucket.from_data(
Expand Down
4 changes: 4 additions & 0 deletions src/guidellm/utils/__init__.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from .auto_importer import AutoImporterMixin
from .console import Colors, Console, ConsoleUpdateStep, StatusIcons, StatusStyles
from .default_group import DefaultGroupHandler
from .dict import recursive_key_update
from .encoding import (
Encoder,
EncodingTypesAlias,
Expand Down Expand Up @@ -55,6 +56,7 @@
)
from .text import (
EndlessTextCreator,
camelize_str,
clean_text,
filter_text,
format_value_display,
Expand All @@ -79,6 +81,7 @@
"EndlessTextCreator",
"InfoMixin",
"IntegerRangeSampler",
"camelize_str",
"InterProcessMessaging",
"InterProcessMessagingManagerQueue",
"InterProcessMessagingPipe",
Expand Down Expand Up @@ -110,6 +113,7 @@
"format_value_display",
"get_literal_vals",
"is_punctuation",
"recursive_key_update",
"load_text",
"safe_add",
"safe_divide",
Expand Down
23 changes: 23 additions & 0 deletions src/guidellm/utils/dict.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
def recursive_key_update(d, key_update_func):
if not isinstance(d, dict) and not isinstance(d, list):
return d

if isinstance(d, list):
for item in d:
recursive_key_update(item, key_update_func)
return d

updated_key_pairs = []
for key, _ in d.items():
updated_key = key_update_func(key)
if key != updated_key:
updated_key_pairs.append((key, updated_key))

for key_pair in updated_key_pairs:
old_key, updated_key = key_pair
d[updated_key] = d[old_key]
del d[old_key]

for _, value in d.items():
recursive_key_update(value, key_update_func)
return d
7 changes: 7 additions & 0 deletions src/guidellm/utils/text.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
__all__ = [
"MAX_PATH_LENGTH",
"EndlessTextCreator",
"camelize_str",
"clean_text",
"filter_text",
"format_value_display",
Expand Down Expand Up @@ -281,6 +282,12 @@ def is_punctuation(text: str) -> bool:
return len(text) == 1 and not text.isalnum() and not text.isspace()


def camelize_str(snake_case_string: str) -> str:
return (words := snake_case_string.split("_"))[0].lower() + "".join(
word.capitalize() for word in words[1:]
)


class EndlessTextCreator:
"""
Infinite text generator for load testing and content creation operations.
Expand Down
71 changes: 71 additions & 0 deletions tests/unit/utils/dict.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
import pytest

from guidellm.utils.dict import recursive_key_update


def update_str(string):
return string + "_updated"


@pytest.mark.smoke
def test_recursive_key_update_updates_keys():
my_dict = {
"my_key": {
"my_nested_key": {"my_double_nested_key": "someValue"},
"my_other_nested_key": "someValue",
},
"my_other_key": "value",
}
my_updated_dict = {
"my_key_updated": {
"my_nested_key_updated": {"my_double_nested_key_updated": "someValue"},
"my_other_nested_key_updated": "someValue",
},
"my_other_key_updated": "value",
}
recursive_key_update(my_dict, update_str)
assert my_dict == my_updated_dict


def truncate_str_to_ten(string):
return string[:10]


@pytest.mark.smoke
def test_recursive_key_update_leaves_unchanged_keys():
my_dict = {
"my_key": {
"my_nested_key": {"my_double_nested_key": "someValue"},
"my_other_nested_key": "someValue",
},
"my_other_key": "value",
}
my_updated_dict = {
"my_key": {
"my_nested_": {"my_double_": "someValue"},
"my_other_n": "someValue",
},
"my_other_k": "value",
}
recursive_key_update(my_dict, truncate_str_to_ten)
assert my_dict == my_updated_dict


@pytest.mark.smoke
def test_recursive_key_update_updates_dicts_in_list():
my_dict = {
"my_key": [
{"my_list_item_key_1": "someValue"},
{"my_list_item_key_2": "someValue"},
{"my_list_item_key_3": "someValue"},
]
}
my_updated_dict = {
"my_key_updated": [
{"my_list_item_key_1_updated": "someValue"},
{"my_list_item_key_2_updated": "someValue"},
{"my_list_item_key_3_updated": "someValue"},
]
}
recursive_key_update(my_dict, update_str)
assert my_dict == my_updated_dict
13 changes: 13 additions & 0 deletions tests/unit/utils/text.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
import pytest

from guidellm.utils.text import camelize_str


@pytest.mark.smoke
def test_camelize_str_camelizes_string():
assert camelize_str("no_longer_snake_case") == "noLongerSnakeCase"


@pytest.mark.smoke
def test_camelize_str_leaves_non_snake_case_text_untouched():
assert camelize_str("notsnakecase") == "notsnakecase"
Loading