Trying one more time for quality

Author: markurtz

src/guidellm/benchmark/benchmarker.py

@@ -14,7 +14,7 @@
 )
 
 from pydantic import Field
-from transformers import PreTrainedTokenizer  # type: ignore  # noqa: PGH003
+from transformers import PreTrainedTokenizerBase  # type: ignore  # noqa: PGH003
 
 from guidellm.backend import Backend, ResponseSummary
 from guidellm.benchmark.aggregator import (
@@ -295,7 +295,7 @@ def __init__(
         request_loader: Iterable[GenerationRequest],
         request_loader_description: GenerativeRequestLoaderDescription,
         benchmark_save_extras: Optional[Dict[str, Any]] = None,
-        processor: Optional[Union[str, Path, PreTrainedTokenizer]] = None,
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]] = None,
         processor_args: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(

src/guidellm/request/loader.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 from typing import (
     Any,
-    Callable,
     Dict,
     Iterable,
     Iterator,
@@ -13,7 +12,7 @@
 )
 
 from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
-from transformers import PreTrainedTokenizer  # type: ignore[import]
+from transformers import PreTrainedTokenizerBase  # type: ignore[import]
 
 from guidellm.dataset import ColumnInputTypes, load_dataset
 from guidellm.objects import StandardBaseModel
@@ -80,7 +79,7 @@ def __init__(
             IterableDatasetDict,
         ],
         data_args: Optional[Dict[str, Any]],
-        processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]],
+        processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
         processor_args: Optional[Dict[str, Any]],
         shuffle: bool = True,
         iter_type: Literal["finite", "infinite"] = "finite",