diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index 95b51d70..28b983fb 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -5,6 +5,7 @@ import math from abc import ABC, abstractmethod from collections import OrderedDict +from copy import deepcopy from datetime import datetime from pathlib import Path from typing import Any, ClassVar @@ -36,6 +37,7 @@ safe_format_timestamp, split_text_list_by_length, ) +from guidellm.utils import recursive_key_update, camelize_str __all__ = [ "GenerativeBenchmarkerCSV", @@ -711,8 +713,6 @@ async def finalize(self, report: GenerativeBenchmarksReport) -> Path: :param report: The completed benchmark report. :return: Path to the saved HTML file. """ - import humps - output_path = self.output_path if output_path.is_dir(): output_path = output_path / GenerativeBenchmarkerHTML.DEFAULT_FILE @@ -720,13 +720,13 @@ async def finalize(self, report: GenerativeBenchmarksReport) -> Path: data_builder = UIDataBuilder(report.benchmarks) data = data_builder.to_dict() - camel_data = humps.camelize(data) + camel_data = recursive_key_update(deepcopy(data), camelize_str) ui_api_data = {} - for key, value in camel_data.items(): - placeholder_key = f"window.{humps.decamelize(key)} = {{}};" + for k, v in camel_data.items(): + placeholder_key = f"window.{k} = {{}};" replacement_value = ( - f"window.{humps.decamelize(key)} = {json.dumps(value, indent=2)};\n" + f"window.{k} = {json.dumps(v, indent=2)};\n" ) ui_api_data[placeholder_key] = replacement_value diff --git a/src/guidellm/presentation/data_models.py b/src/guidellm/presentation/data_models.py index 9036636a..c1e8f13f 100644 --- a/src/guidellm/presentation/data_models.py +++ b/src/guidellm/presentation/data_models.py @@ -67,7 +67,7 @@ class RunInfo(BaseModel): @classmethod def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]): - model = benchmarks[0].worker.backend_model or "N/A" + model = benchmarks[0].benchmarker.backend.get("model", "N/A") timestamp = max( bm.run_stats.start_time for bm in benchmarks if bm.start_time is not None ) @@ -108,8 +108,8 @@ class WorkloadDetails(BaseModel): @classmethod def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]): - target = benchmarks[0].worker.backend_target - rate_type = benchmarks[0].args.profile.type_ + target = benchmarks[0].benchmarker.backend.get("target", "N/A") + rate_type = benchmarks[0].scheduler.strategy.type_ successful_requests = [ req for bm in benchmarks for req in bm.requests.successful ] @@ -152,13 +152,13 @@ def from_benchmarks(cls, benchmarks: list["GenerativeBenchmark"]): statistics=output_token_stats, buckets=output_token_buckets, bucket_width=1 ) - min_start_time = benchmarks[0].run_stats.start_time + min_start_time = benchmarks[0].start_time all_req_times = [ - req.start_time - min_start_time + req.scheduler_info.started_at - min_start_time for bm in benchmarks for req in bm.requests.successful - if req.start_time is not None + if req.scheduler_info.started_at is not None ] number_of_buckets = len(benchmarks) request_over_time_buckets, bucket_width = Bucket.from_data( diff --git a/src/guidellm/utils/__init__.py b/src/guidellm/utils/__init__.py index 20daeea4..bd6b5a90 100644 --- a/src/guidellm/utils/__init__.py +++ b/src/guidellm/utils/__init__.py @@ -1,6 +1,7 @@ from .auto_importer import AutoImporterMixin from .console import Colors, Console, ConsoleUpdateStep, StatusIcons, StatusStyles from .default_group import DefaultGroupHandler +from .dict import recursive_key_update from .encoding import ( Encoder, EncodingTypesAlias, @@ -55,6 +56,7 @@ ) from .text import ( EndlessTextCreator, + camelize_str, clean_text, filter_text, format_value_display, @@ -79,6 +81,7 @@ "EndlessTextCreator", "InfoMixin", "IntegerRangeSampler", + "camelize_str", "InterProcessMessaging", "InterProcessMessagingManagerQueue", "InterProcessMessagingPipe", @@ -110,6 +113,7 @@ "format_value_display", "get_literal_vals", "is_punctuation", + "recursive_key_update", "load_text", "safe_add", "safe_divide", diff --git a/src/guidellm/utils/dict.py b/src/guidellm/utils/dict.py new file mode 100644 index 00000000..5b4579c9 --- /dev/null +++ b/src/guidellm/utils/dict.py @@ -0,0 +1,23 @@ +def recursive_key_update(d, key_update_func): + if not isinstance(d, dict) and not isinstance(d, list): + return d + + if isinstance(d, list): + for item in d: + recursive_key_update(item, key_update_func) + return d + + updated_key_pairs = [] + for key, _ in d.items(): + updated_key = key_update_func(key) + if key != updated_key: + updated_key_pairs.append((key, updated_key)) + + for key_pair in updated_key_pairs: + old_key, updated_key = key_pair + d[updated_key] = d[old_key] + del d[old_key] + + for _, value in d.items(): + recursive_key_update(value, key_update_func) + return d diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index 8385ec7b..a659ac6a 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -28,6 +28,7 @@ __all__ = [ "MAX_PATH_LENGTH", "EndlessTextCreator", + "camelize_str", "clean_text", "filter_text", "format_value_display", @@ -281,6 +282,12 @@ def is_punctuation(text: str) -> bool: return len(text) == 1 and not text.isalnum() and not text.isspace() +def camelize_str(snake_case_string: str) -> str: + return (words := snake_case_string.split("_"))[0].lower() + "".join( + word.capitalize() for word in words[1:] + ) + + class EndlessTextCreator: """ Infinite text generator for load testing and content creation operations. diff --git a/tests/unit/utils/dict.py b/tests/unit/utils/dict.py new file mode 100644 index 00000000..09d93df6 --- /dev/null +++ b/tests/unit/utils/dict.py @@ -0,0 +1,71 @@ +import pytest + +from guidellm.utils.dict import recursive_key_update + + +def update_str(string): + return string + "_updated" + + +@pytest.mark.smoke +def test_recursive_key_update_updates_keys(): + my_dict = { + "my_key": { + "my_nested_key": {"my_double_nested_key": "someValue"}, + "my_other_nested_key": "someValue", + }, + "my_other_key": "value", + } + my_updated_dict = { + "my_key_updated": { + "my_nested_key_updated": {"my_double_nested_key_updated": "someValue"}, + "my_other_nested_key_updated": "someValue", + }, + "my_other_key_updated": "value", + } + recursive_key_update(my_dict, update_str) + assert my_dict == my_updated_dict + + +def truncate_str_to_ten(string): + return string[:10] + + +@pytest.mark.smoke +def test_recursive_key_update_leaves_unchanged_keys(): + my_dict = { + "my_key": { + "my_nested_key": {"my_double_nested_key": "someValue"}, + "my_other_nested_key": "someValue", + }, + "my_other_key": "value", + } + my_updated_dict = { + "my_key": { + "my_nested_": {"my_double_": "someValue"}, + "my_other_n": "someValue", + }, + "my_other_k": "value", + } + recursive_key_update(my_dict, truncate_str_to_ten) + assert my_dict == my_updated_dict + + +@pytest.mark.smoke +def test_recursive_key_update_updates_dicts_in_list(): + my_dict = { + "my_key": [ + {"my_list_item_key_1": "someValue"}, + {"my_list_item_key_2": "someValue"}, + {"my_list_item_key_3": "someValue"}, + ] + } + my_updated_dict = { + "my_key_updated": [ + {"my_list_item_key_1_updated": "someValue"}, + {"my_list_item_key_2_updated": "someValue"}, + {"my_list_item_key_3_updated": "someValue"}, + ] + } + recursive_key_update(my_dict, update_str) + assert my_dict == my_updated_dict diff --git a/tests/unit/utils/text.py b/tests/unit/utils/text.py new file mode 100644 index 00000000..ae0fa52f --- /dev/null +++ b/tests/unit/utils/text.py @@ -0,0 +1,13 @@ +import pytest + +from guidellm.utils.text import camelize_str + + +@pytest.mark.smoke +def test_camelize_str_camelizes_string(): + assert camelize_str("no_longer_snake_case") == "noLongerSnakeCase" + + +@pytest.mark.smoke +def test_camelize_str_leaves_non_snake_case_text_untouched(): + assert camelize_str("notsnakecase") == "notsnakecase"