Skip to content

Commit 65becf0

Browse files
authored
Add ut for tf_transformers (#327)
## Summary Add ut for tf_transformers ## Details add ut for tf_transformers from 31% to 100% <img width="1784" height="1428" alt="123667c9d71dcbdc198ebe2b84ba75a6" src="https://github.com/user-attachments/assets/d2626a5f-38d3-4fdc-93b1-550bd4d95089" /> <img width="2398" height="934" alt="aa0509b6f9f134509a0d762f2a218fe4" src="https://github.com/user-attachments/assets/29abc522-70e6-484b-87a4-7cd8b52a3cf7" /> ## Test Plan pytest -v tests/unit/utils/test_hf_transformers.py <img width="1980" height="528" alt="0bd1b89ca91755096c162a4c82c11faa" src="https://github.com/user-attachments/assets/f3df55a3-efcc-468f-987a-f4105c36060f" /> pytest --cov=guidellm tests/unit/utils <img width="1916" height="540" alt="b975004df7a4a63e8272a43d00ca057f" src="https://github.com/user-attachments/assets/db6cbffa-54bc-44a3-aa88-889dc6420207" /> <img width="2398" height="934" alt="aa0509b6f9f134509a0d762f2a218fe4" src="https://github.com/user-attachments/assets/75a50ec6-c99f-4560-bf9d-8b2bcc6eb224" /> ## Related Issues - Resolves # none --- - [x] "I certify that all code in this PR is my own, except as noted below." ## Use of AI - [x] Includes AI-assisted code completion - [ ] Includes code generated by an AI application - [ ] Includes AI-generated tests (NOTE: AI written tests should have a docstring that includes `## WRITTEN BY AI ##`)
2 parents 5859175 + a0fd2e0 commit 65becf0

File tree

1 file changed

+40
-0
lines changed

1 file changed

+40
-0
lines changed
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
import pytest
2+
import transformers
3+
from transformers import PreTrainedTokenizerBase
4+
5+
from guidellm.utils.hf_transformers import check_load_processor
6+
7+
8+
class DummyTokenizer(PreTrainedTokenizerBase):
9+
pass
10+
11+
12+
def test_processor_is_none():
13+
with pytest.raises(ValueError, match="Processor/Tokenizer is required for test."):
14+
check_load_processor(None, None, "test")
15+
16+
17+
def test_processor_not_isinstance():
18+
with pytest.raises(ValueError, match="Invalid processor/Tokenizer for test."):
19+
check_load_processor(123, None, "test") # type: ignore
20+
21+
22+
def test_processor_load_by_path(monkeypatch, tmp_path):
23+
monkeypatch.setattr(
24+
transformers.AutoTokenizer,
25+
"from_pretrained",
26+
lambda *args, **kwargs: DummyTokenizer(),
27+
)
28+
tokenizer = check_load_processor(tmp_path, None, "test")
29+
assert isinstance(tokenizer, PreTrainedTokenizerBase)
30+
31+
32+
def test_processor_load_error(monkeypatch):
33+
def raise_error(*args, **kwargs):
34+
raise RuntimeError("test error")
35+
36+
monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", raise_error)
37+
with pytest.raises(
38+
ValueError, match="Failed to load processor/Tokenizer for test."
39+
):
40+
check_load_processor("gpt2", None, "test")

0 commit comments

Comments
 (0)