Skip to content

Commit cd4e62d

Browse files
jaredoconnelljaredoconnell
committed
Added option to regenerate test artifact
1 parent 1e496b0 commit cd4e62d

File tree

2 files changed

+31
-38
lines changed

2 files changed

+31
-38
lines changed

‎tests/unit/entrypoints/assets/benchmarks_stripped_output.txt‎

Lines changed: 19 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -3,44 +3,29 @@
33
Benchmarks Metadata:
44
Run id:93e36b31-b454-471d-ba62-6b2671585485
55
Duration:30.2 seconds
6-
Profile:type=sweep, strategies=['synchronous', 'throughput', 'constant', 'constant', 'constant', 'constant',
7-
'constant', 'constant', 'constant', 'constant'], max_concurrency=None
8-
Args:max_number=None, max_duration=30.0, warmup_number=None, warmup_duration=None, cooldown_number=None,
9-
cooldown_duration=None
10-
Worker:type_='generative_requests_worker' backend_type='openai_http' backend_target='example_target'
11-
backend_model='example_model' backend_info={'max_output_tokens': 16384, 'timeout': 300, 'http2': True,
12-
'authorization': False, 'organization': None, 'project': None, 'text_completions_path': '/v1/completions',
13-
'chat_completions_path': '/v1/chat/completions'}
14-
Request Loader:type_='generative_request_loader' data='prompt_tokens=256,output_tokens=128' data_args=None
15-
processor='example_processor' processor_args=None
6+
Profile:type=sweep, strategies=['synchronous', 'throughput', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant', 'constant'],
7+
max_concurrency=None
8+
Args:max_number=None, max_duration=30.0, warmup_number=None, warmup_duration=None, cooldown_number=None, cooldown_duration=None
9+
Worker:type_='generative_requests_worker' backend_type='openai_http' backend_target='example_target' backend_model='example_model' backend_info={'max_output_tokens': 16384,
10+
'timeout': 300, 'http2': True, 'authorization': False, 'organization': None, 'project': None, 'text_completions_path': '/v1/completions', 'chat_completions_path':
11+
'/v1/chat/completions'}
12+
Request Loader:type_='generative_request_loader' data='prompt_tokens=256,output_tokens=128' data_args=None processor='example_processor' processor_args=None
1613
Extras:None
1714

1815

1916
Benchmarks Info:
20-
========================================================================================================================
21-
===========================
22-
Metadata |||| Requests Made ||| Prompt Tok/Req ||| Output Tok/Req ||| Prompt Tok
23-
Total||| Output Tok Total ||
24-
Benchmark| Start Time| End Time| Duration (s)| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc|
25-
Err| Comp| Inc| Err
26-
-----------|-----------|---------|-------------|------|-----|-----|------|------|----|-------|-----|-----|-------|-----|
27-
-----|-------|------|------
28-
synchronous| 16:59:28| 16:59:58| 30.0| 46| 1| 0| 257.1| 256.0| 0.0| 128.0| 0.0| 0.0| 11827| 256|
29-
0| 5888| 0| 0
30-
========================================================================================================================
31-
===========================
17+
===================================================================================================================================================
18+
Metadata |||| Requests Made ||| Prompt Tok/Req ||| Output Tok/Req ||| Prompt Tok Total||| Output Tok Total ||
19+
Benchmark| Start Time| End Time| Duration (s)| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err| Comp| Inc| Err
20+
-----------|-----------|---------|-------------|------|-----|-----|------|------|----|-------|-----|-----|-------|-----|-----|-------|------|------
21+
synchronous| 16:59:28| 16:59:58| 30.0| 46| 1| 0| 257.1| 256.0| 0.0| 128.0| 0.0| 0.0| 11827| 256| 0| 5888| 0| 0
22+
===================================================================================================================================================
3223

3324

3425
Benchmarks Stats:
35-
========================================================================================================================
36-
=======================
37-
Metadata | Request Stats || Out Tok/sec| Tot Tok/sec| Req Latency (sec) ||| TTFT (ms) ||| ITL (ms)
38-
||| TPOT (ms) ||
39-
Benchmark| Per Second| Concurrency| mean| mean| mean| median| p99| mean| median| p99| mean| median|
40-
p99| mean| median| p99
41-
-----------|-----------|------------|------------|------------|------|--------|------|-----|-------|-----|-----|-------|
42-
----|-----|-------|----
43-
synchronous| 1.55| 1.00| 198.1| 992.7| 0.64| 0.64| 0.69| 16.8| 16.4| 21.3| 4.9| 4.9|
44-
5.3| 4.9| 4.9| 5.2
45-
========================================================================================================================
46-
=======================
26+
===============================================================================================================================================
27+
Metadata | Request Stats || Out Tok/sec| Tot Tok/sec| Req Latency (sec) ||| TTFT (ms) ||| ITL (ms) ||| TPOT (ms) ||
28+
Benchmark| Per Second| Concurrency| mean| mean| mean| median| p99| mean| median| p99| mean| median| p99| mean| median| p99
29+
-----------|-----------|------------|------------|------------|------|--------|------|-----|-------|-----|-----|-------|----|-----|-------|----
30+
synchronous| 1.55| 1.00| 198.1| 992.7| 0.64| 0.64| 0.69| 16.8| 16.4| 21.3| 4.9| 4.9| 5.3| 4.9| 4.9| 5.2
31+
===============================================================================================================================================

‎tests/unit/entrypoints/test_display_entrypoint.py‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
from guidellm.benchmark import display_benchmarks_report
88

99

10+
# Set to true to re-write the expected output.
11+
REGENERATE_ARTIFACTS = False
12+
13+
1014
@pytest.fixture
1115
def get_test_asset_dir():
1216
def _() -> Path:
@@ -32,14 +36,18 @@ def test_display_entrypoint_yaml(capfd, get_test_asset_dir):
3236

3337

3438
def generic_test_display_entrypoint(filename, capfd, get_test_asset_dir):
35-
os.environ["COLUMNS"] = "120" # CLI output depends on terminal width.
39+
os.environ["COLUMNS"] = "180" # CLI output depends on terminal width.
3640
asset_dir = get_test_asset_dir()
3741
display_benchmarks_report(asset_dir / filename)
3842
out, err = capfd.readouterr()
3943
expected_output_path = asset_dir / "benchmarks_stripped_output.txt"
40-
with expected_output_path.open(encoding="utf_8") as file:
41-
expected_output = file.read()
42-
assert out == expected_output
44+
if REGENERATE_ARTIFACTS:
45+
expected_output_path.write_text(out)
46+
assert False # Fail to prevent accidentally leaving this set
47+
else:
48+
with expected_output_path.open(encoding="utf_8") as file:
49+
expected_output = file.read()
50+
assert out == expected_output
4351

4452

4553
if __name__ == "__main__":

0 commit comments

Comments
 (0)