| 
 | 1 | +import pytest  | 
 | 2 | +import transformers  | 
 | 3 | +from transformers import PreTrainedTokenizerBase  | 
 | 4 | + | 
 | 5 | +from guidellm.utils.hf_transformers import check_load_processor  | 
 | 6 | + | 
 | 7 | + | 
 | 8 | +class DummyTokenizer(PreTrainedTokenizerBase):  | 
 | 9 | +    pass  | 
 | 10 | + | 
 | 11 | + | 
 | 12 | +def test_processor_is_none():  | 
 | 13 | +    with pytest.raises(ValueError, match="Processor/Tokenizer is required for test."):  | 
 | 14 | +        check_load_processor(None, None, "test")  | 
 | 15 | + | 
 | 16 | + | 
 | 17 | +def test_processor_not_isinstance():  | 
 | 18 | +    with pytest.raises(ValueError, match="Invalid processor/Tokenizer for test."):  | 
 | 19 | +        check_load_processor(123, None, "test")  # type: ignore  | 
 | 20 | + | 
 | 21 | + | 
 | 22 | +def test_processor_load_by_path(monkeypatch, tmp_path):  | 
 | 23 | +    monkeypatch.setattr(  | 
 | 24 | +        transformers.AutoTokenizer,  | 
 | 25 | +        "from_pretrained",  | 
 | 26 | +        lambda *args, **kwargs: DummyTokenizer(),  | 
 | 27 | +    )  | 
 | 28 | +    tokenizer = check_load_processor(tmp_path, None, "test")  | 
 | 29 | +    assert isinstance(tokenizer, PreTrainedTokenizerBase)  | 
 | 30 | + | 
 | 31 | + | 
 | 32 | +def test_processor_load_error(monkeypatch):  | 
 | 33 | +    def raise_error(*args, **kwargs):  | 
 | 34 | +        raise RuntimeError("test error")  | 
 | 35 | + | 
 | 36 | +    monkeypatch.setattr("transformers.AutoTokenizer.from_pretrained", raise_error)  | 
 | 37 | +    with pytest.raises(  | 
 | 38 | +        ValueError, match="Failed to load processor/Tokenizer for test."  | 
 | 39 | +    ):  | 
 | 40 | +        check_load_processor("gpt2", None, "test")  | 
0 commit comments