Just kidding, let's try that again and hopefully fix quality and tests

Author: markurtz

src/guidellm/benchmark/entrypoints.py

@@ -1,8 +1,10 @@
 from pathlib import Path
-from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Union
+from typing import Any, Dict, Iterable, List, Literal, Optional, Union
 
 from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict
-from transformers import PreTrainedTokenizer  # type: ignore[import]
+from transformers import (  # type: ignore[import]
+    PreTrainedTokenizerBase,
+)
 
 from guidellm.backend import Backend, BackendType
 from guidellm.benchmark.benchmark import GenerativeBenchmark
@@ -22,7 +24,7 @@ async def benchmark_generative_text(
     backend_type: BackendType,
     backend_args: Optional[Dict[str, Any]],
     model: Optional[str],
-    processor: Optional[Union[str, Path, PreTrainedTokenizer, Callable]],
+    processor: Optional[Optional[Union[str, Path, PreTrainedTokenizerBase]]],
     processor_args: Optional[Dict[str, Any]],
     data: Union[
         str,

src/guidellm/dataset/entrypoints.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple, Union
 
 from datasets import Dataset, IterableDataset
@@ -15,7 +16,7 @@
 def load_dataset(
     data: Any,
     data_args: Optional[Dict[str, Any]],
-    processor: PreTrainedTokenizerBase,
+    processor: Optional[Union[str, Path, PreTrainedTokenizerBase]],
     processor_args: Optional[Dict[str, Any]],
     random_seed: int = 42,
     split_pref_order: Optional[List[str]] = None,

src/guidellm/objects/statistics.py

@@ -117,14 +117,14 @@ def from_distribution_function(
         :return: An instance of DistributionSummary with calculated values.
         """
         values, weights = zip(*distribution) if distribution else ([], [])
-        values = np.array(values)
-        weights = np.array(weights)
+        values = np.array(values)  # type: ignore[assignment]
+        weights = np.array(weights)  # type: ignore[assignment]
 
         # create the PDF
         probabilities = weights / np.sum(weights)  # type: ignore[operator]
         pdf = np.column_stack((values, probabilities))
         pdf = pdf[np.argsort(pdf[:, 0])]
-        values = pdf[:, 0]
+        values = pdf[:, 0]  # type: ignore[assignment]
         probabilities = pdf[:, 1]
 
         # calculate the CDF

tests/e2e/test_placeholder.py

@@ -1,2 +1,6 @@
+import pytest
+
+
+@pytest.mark.smoke()
 def test_placeholder():
     assert True

tests/integration/test_placeholder.py

@@ -1,2 +1,6 @@
+import pytest
+
+
+@pytest.mark.smoke()
 def test_placeholder():
     assert True