diff --git a/pyproject.toml b/pyproject.toml index 984f44ae..df91260e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,8 @@ name = "guidellm" description = "Guidance platform for deploying and managing large language models." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.9.0,<4.0" -license = "Apache-2.0" -license-files = ["LICENSE"] -authors = [ { name = "Red Hat" } ] +license = { text = "Apache-2.0" } +authors = [{ name = "Red Hat" }] keywords = [ "ai", "benchmarking", @@ -47,10 +46,14 @@ keywords = [ ] dependencies = [ "click>=8.0.0,<8.2.0", + "culsans~=0.9.0", "datasets", + "eval_type_backport", + "faker", "ftfy>=6.0.0", "httpx[http2]<1.0.0", "loguru", + "msgpack", "numpy", "pillow", "protobuf", @@ -58,7 +61,9 @@ dependencies = [ "pydantic-settings>=2.0.0", "pyyaml>=6.0.0", "rich", + "sanic", "transformers", + "uvloop>=0.18", ] [project.optional-dependencies] @@ -81,7 +86,7 @@ dev = [ # testing "lorem~=0.1.1", "pytest~=8.2.2", - "pytest-asyncio~=0.23.8", + "pytest-asyncio~=1.1.0", "pytest-cov~=5.0.0", "pytest-mock~=3.14.0", "pytest-rerunfailures~=14.0", @@ -143,11 +148,17 @@ exclude = ["venv", ".tox"] follow_imports = 'silent' [[tool.mypy.overrides]] -module = ["datasets.*", "transformers.*", "setuptools.*", "setuptools_git_versioning.*"] -ignore_missing_imports=true +module = [ + "datasets.*", + "transformers.*", + "setuptools.*", + "setuptools_git_versioning.*", +] +ignore_missing_imports = true [tool.ruff] +target-version = "py39" line-length = 88 indent-width = 4 exclude = ["build", "dist", "env", ".venv"] @@ -158,64 +169,65 @@ indent-style = "space" [tool.ruff.lint] ignore = [ - "PLR0913", - "TC001", - "COM812", - "ISC001", - "TC002", + "COM812", # ignore trailing comma errors due to older Python versions + "PD011", # ignore .values usage since ruff assumes it's a Pandas DataFrame + "PLR0913", # ignore too many arguments in function definitions "PLW1514", # allow Path.open without encoding - "RET505", # allow `else` blocks - "RET506", # allow `else` blocks - "PD011", # ignore .values usage since ruff assumes it's a Pandas DataFrame + "RET505", # allow `else` blocks + "RET506", # allow `else` blocks + "S311", # allow standard pseudo-random generators + "TC001", # ignore imports used only for type checking + "TC002", # ignore imports used only for type checking + "TC003", # ignore imports used only for type checking ] select = [ # Rules reference: https://docs.astral.sh/ruff/rules/ # Code Style / Formatting - "E", # pycodestyle: checks adherence to PEP 8 conventions including spacing, indentation, and line length - "W", # pycodestyle: checks adherence to PEP 8 conventions including spacing, indentation, and line length - "A", # flake8-builtins: prevents shadowing of Python built-in names - "C", # Convention: ensures code adheres to specific style and formatting conventions - "COM", # flake8-commas: enforces the correct use of trailing commas - "ERA", # eradicate: detects commented-out code that should be removed - "I", # isort: ensures imports are sorted in a consistent manner - "ICN", # flake8-import-conventions: enforces import conventions for better readability - "N", # pep8-naming: enforces PEP 8 naming conventions for classes, functions, and variables - "NPY", # NumPy: enforces best practices for using the NumPy library - "PD", # pandas-vet: enforces best practices for using the pandas library - "PT", # flake8-pytest-style: enforces best practices and style conventions for pytest tests - "PTH", # flake8-use-pathlib: encourages the use of pathlib over os.path for file system operations - "Q", # flake8-quotes: enforces consistent use of single or double quotes - "TCH", # flake8-type-checking: enforces type checking practices and standards - "TID", # flake8-tidy-imports: enforces tidy and well-organized imports + "E", # pycodestyle: checks adherence to PEP 8 conventions including spacing, indentation, and line length + "W", # pycodestyle: checks adherence to PEP 8 conventions including spacing, indentation, and line length + "A", # flake8-builtins: prevents shadowing of Python built-in names + "C", # Convention: ensures code adheres to specific style and formatting conventions + "COM", # flake8-commas: enforces the correct use of trailing commas + "ERA", # eradicate: detects commented-out code that should be removed + "I", # isort: ensures imports are sorted in a consistent manner + "ICN", # flake8-import-conventions: enforces import conventions for better readability + "N", # pep8-naming: enforces PEP 8 naming conventions for classes, functions, and variables + "NPY", # NumPy: enforces best practices for using the NumPy library + "PD", # pandas-vet: enforces best practices for using the pandas library + "PT", # flake8-pytest-style: enforces best practices and style conventions for pytest tests + "PTH", # flake8-use-pathlib: encourages the use of pathlib over os.path for file system operations + "Q", # flake8-quotes: enforces consistent use of single or double quotes + "TCH", # flake8-type-checking: enforces type checking practices and standards + "TID", # flake8-tidy-imports: enforces tidy and well-organized imports "RUF022", # flake8-ruff: enforce sorting of __all__ in modules # Code Structure / Complexity - "C4", # flake8-comprehensions: improves readability and performance of list, set, and dict comprehensions + "C4", # flake8-comprehensions: improves readability and performance of list, set, and dict comprehensions "C90", # mccabe: checks for overly complex code using cyclomatic complexity "ISC", # flake8-implicit-str-concat: prevents implicit string concatenation "PIE", # flake8-pie: identifies and corrects common code inefficiencies and mistakes - "R", # Refactor: suggests improvements to code structure and readability + "R", # Refactor: suggests improvements to code structure and readability "SIM", # flake8-simplify: simplifies complex expressions and improves code readability # Code Security / Bug Prevention - "ARG", # flake8-unused-arguments: detects unused function and method arguments + "ARG", # flake8-unused-arguments: detects unused function and method arguments "ASYNC", # flake8-async: identifies incorrect or inefficient usage patterns in asynchronous code - "B", # flake8-bugbear: detects common programming mistakes and potential bugs - "BLE", # flake8-blind-except: prevents blind exceptions that catch all exceptions without handling - "E", # Error: detects and reports errors in the code - "F", # Pyflakes: detects unused imports, shadowed imports, undefined variables, and various formatting errors in string operations - "INP", # flake8-no-pep420: prevents implicit namespace packages by requiring __init__.py - "PGH", # pygrep-hooks: detects deprecated and dangerous code patterns - "PL", # Pylint: comprehensive source code analyzer for enforcing coding standards and detecting errors - "RSE", # flake8-raise: ensures exceptions are raised correctly - "S", # flake8-bandit: detects security issues and vulnerabilities in the code - "SLF", # flake8-self: prevents incorrect usage of the self argument in class methods - "T10", # flake8-debugger: detects the presence of debugging tools such as pdb - "T20", # flake8-print: detects print statements left in the code - "UP", # pyupgrade: automatically upgrades syntax for newer versions of Python - "W", # Warning: provides warnings about potential issues in the code - "YTT", # flake8-2020: identifies code that will break with future Python releases + "B", # flake8-bugbear: detects common programming mistakes and potential bugs + "BLE", # flake8-blind-except: prevents blind exceptions that catch all exceptions without handling + "E", # Error: detects and reports errors in the code + "F", # Pyflakes: detects unused imports, shadowed imports, undefined variables, and various formatting errors in string operations + "INP", # flake8-no-pep420: prevents implicit namespace packages by requiring __init__.py + "PGH", # pygrep-hooks: detects deprecated and dangerous code patterns + "PL", # Pylint: comprehensive source code analyzer for enforcing coding standards and detecting errors + "RSE", # flake8-raise: ensures exceptions are raised correctly + "S", # flake8-bandit: detects security issues and vulnerabilities in the code + "SLF", # flake8-self: prevents incorrect usage of the self argument in class methods + "T10", # flake8-debugger: detects the presence of debugging tools such as pdb + "T20", # flake8-print: detects print statements left in the code + "UP", # pyupgrade: automatically upgrades syntax for newer versions of Python + "W", # Warning: provides warnings about potential issues in the code + "YTT", # flake8-2020: identifies code that will break with future Python releases # Code Documentation "FIX", # flake8-fixme: detects FIXMEs and other temporary comments that should be resolved @@ -223,17 +235,17 @@ select = [ [tool.ruff.lint.extend-per-file-ignores] "tests/**/*.py" = [ - "S101", # asserts allowed in tests - "ARG", # Unused function args allowed in tests + "S101", # asserts allowed in tests + "ARG", # Unused function args allowed in tests "PLR2004", # Magic value used in comparison - "TCH002", # No import only type checking in tests - "SLF001", # enable private member access in tests - "S105", # allow hardcoded passwords in tests - "S311", # allow standard pseudo-random generators in tests - "PT011", # allow generic exceptions in tests - "N806", # allow uppercase variable names in tests - "PGH003", # allow general ignores in tests - "S106", # allow hardcoded passwords in tests + "TCH002", # No import only type checking in tests + "SLF001", # enable private member access in tests + "S105", # allow hardcoded passwords in tests + "S311", # allow standard pseudo-random generators in tests + "PT011", # allow generic exceptions in tests + "N806", # allow uppercase variable names in tests + "PGH003", # allow general ignores in tests + "S106", # allow hardcoded passwords in tests "PLR0915", # allow complext statements in tests ] @@ -246,5 +258,5 @@ addopts = '-s -vvv --cache-clear' markers = [ "smoke: quick tests to check basic functionality", "sanity: detailed tests to ensure major functions work correctly", - "regression: tests to ensure that new changes do not break existing functionality" + "regression: tests to ensure that new changes do not break existing functionality", ] diff --git a/setup.py b/setup.py index 009cf362..623bad28 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ from setuptools import setup from setuptools_git_versioning import count_since, get_branch, get_sha, get_tags -LAST_RELEASE_VERSION = Version("0.0.0") +LAST_RELEASE_VERSION = Version("0.3.0") TAG_VERSION_PATTERN = re.compile(r"^v(\d+\.\d+\.\d+)$") diff --git a/src/guidellm/__init__.py b/src/guidellm/__init__.py index 9333860e..f2206e94 100644 --- a/src/guidellm/__init__.py +++ b/src/guidellm/__init__.py @@ -20,7 +20,8 @@ hf_logging.set_verbosity_error() logging.getLogger("transformers").setLevel(logging.ERROR) -from .config import ( +from .logger import configure_logger, logger +from .settings import ( DatasetSettings, Environment, LoggingSettings, @@ -30,7 +31,6 @@ reload_settings, settings, ) -from .logger import configure_logger, logger __all__ = [ "DatasetSettings", diff --git a/src/guidellm/__main__.py b/src/guidellm/__main__.py index 7cba6a7c..f82c19cf 100644 --- a/src/guidellm/__main__.py +++ b/src/guidellm/__main__.py @@ -13,9 +13,9 @@ ) from guidellm.benchmark.entrypoints import benchmark_with_scenario from guidellm.benchmark.scenario import GenerativeTextScenario, get_builtin_scenarios -from guidellm.config import print_config from guidellm.preprocess.dataset import ShortPromptStrategy, process_dataset from guidellm.scheduler import StrategyType +from guidellm.settings import print_config from guidellm.utils import DefaultGroupHandler from guidellm.utils import cli as cli_tools diff --git a/src/guidellm/backend/backend.py b/src/guidellm/backend/backend.py index bf2788a7..ceffdc77 100644 --- a/src/guidellm/backend/backend.py +++ b/src/guidellm/backend/backend.py @@ -7,7 +7,7 @@ from PIL import Image from guidellm.backend.response import ResponseSummary, StreamingTextResponse -from guidellm.config import settings +from guidellm.settings import settings __all__ = [ "Backend", diff --git a/src/guidellm/backend/openai.py b/src/guidellm/backend/openai.py index 680578cc..e1fcdf89 100644 --- a/src/guidellm/backend/openai.py +++ b/src/guidellm/backend/openai.py @@ -16,7 +16,7 @@ ResponseSummary, StreamingTextResponse, ) -from guidellm.config import settings +from guidellm.settings import settings __all__ = [ "CHAT_COMPLETIONS", diff --git a/src/guidellm/backend/response.py b/src/guidellm/backend/response.py index ee2101d7..f2272a73 100644 --- a/src/guidellm/backend/response.py +++ b/src/guidellm/backend/response.py @@ -2,8 +2,8 @@ from pydantic import computed_field -from guidellm.config import settings from guidellm.objects.pydantic import StandardBaseModel +from guidellm.settings import settings __all__ = [ "RequestArgs", diff --git a/src/guidellm/benchmark/aggregator.py b/src/guidellm/benchmark/aggregator.py index f10eb5ed..9e6ffd68 100644 --- a/src/guidellm/benchmark/aggregator.py +++ b/src/guidellm/benchmark/aggregator.py @@ -21,7 +21,6 @@ GenerativeTextErrorStats, GenerativeTextResponseStats, ) -from guidellm.config import settings from guidellm.objects import ( RunningStats, StandardBaseModel, @@ -40,6 +39,7 @@ SchedulerRequestResult, WorkerDescription, ) +from guidellm.settings import settings from guidellm.utils import check_load_processor __all__ = [ diff --git a/src/guidellm/benchmark/output.py b/src/guidellm/benchmark/output.py index da868106..6759f16f 100644 --- a/src/guidellm/benchmark/output.py +++ b/src/guidellm/benchmark/output.py @@ -20,7 +20,6 @@ SweepProfile, ThroughputProfile, ) -from guidellm.config import settings from guidellm.objects import ( DistributionSummary, StandardBaseModel, @@ -29,6 +28,7 @@ from guidellm.presentation import UIDataBuilder from guidellm.presentation.injector import create_report from guidellm.scheduler import strategy_display_str +from guidellm.settings import settings from guidellm.utils import Colors, split_text_list_by_length from guidellm.utils.dict import recursive_key_update from guidellm.utils.text import camelize_str diff --git a/src/guidellm/benchmark/profile.py b/src/guidellm/benchmark/profile.py index 642cb7a8..ca25fc24 100644 --- a/src/guidellm/benchmark/profile.py +++ b/src/guidellm/benchmark/profile.py @@ -4,7 +4,6 @@ import numpy as np from pydantic import Field, computed_field -from guidellm.config import settings from guidellm.objects import StandardBaseModel from guidellm.scheduler import ( AsyncConstantStrategy, @@ -15,6 +14,7 @@ SynchronousStrategy, ThroughputStrategy, ) +from guidellm.settings import settings __all__ = [ "AsyncProfile", diff --git a/src/guidellm/logger.py b/src/guidellm/logger.py index 527d66ff..da3464f9 100644 --- a/src/guidellm/logger.py +++ b/src/guidellm/logger.py @@ -41,7 +41,7 @@ from loguru import logger -from guidellm.config import LoggingSettings, settings +from guidellm.settings import LoggingSettings, settings __all__ = ["configure_logger", "logger"] @@ -72,7 +72,7 @@ def configure_logger(config: LoggingSettings = settings.logging): sys.stdout, level=config.console_log_level.upper(), format="{time:YY-MM-DD HH:mm:ss}|{level: <8} \ - |{name}:{function}:{line} - {message}" + |{name}:{function}:{line} - {message}", ) if config.log_file or config.log_file_level: diff --git a/src/guidellm/presentation/injector.py b/src/guidellm/presentation/injector.py index 02d53b1d..bb1fd684 100644 --- a/src/guidellm/presentation/injector.py +++ b/src/guidellm/presentation/injector.py @@ -4,7 +4,7 @@ from loguru import logger -from guidellm.config import settings +from guidellm.settings import settings from guidellm.utils.text import load_text diff --git a/src/guidellm/request/loader.py b/src/guidellm/request/loader.py index 48566976..1c875046 100644 --- a/src/guidellm/request/loader.py +++ b/src/guidellm/request/loader.py @@ -11,10 +11,10 @@ from datasets import Dataset, DatasetDict, IterableDataset, IterableDatasetDict from transformers import PreTrainedTokenizerBase # type: ignore[import] -from guidellm.config import settings from guidellm.dataset import ColumnInputTypes, load_dataset from guidellm.objects import StandardBaseModel from guidellm.request.request import GenerationRequest +from guidellm.settings import settings __all__ = [ "GenerativeRequestLoader", diff --git a/src/guidellm/scheduler/scheduler.py b/src/guidellm/scheduler/scheduler.py index 83a611ec..11e1102a 100644 --- a/src/guidellm/scheduler/scheduler.py +++ b/src/guidellm/scheduler/scheduler.py @@ -14,7 +14,6 @@ from loguru import logger -from guidellm.config import settings from guidellm.request.types import ( RequestT, ResponseT, @@ -31,6 +30,7 @@ from guidellm.scheduler.worker import ( RequestsWorker, ) +from guidellm.settings import settings __all__ = ["Scheduler"] diff --git a/src/guidellm/scheduler/strategy.py b/src/guidellm/scheduler/strategy.py index 74d19266..81ff6558 100644 --- a/src/guidellm/scheduler/strategy.py +++ b/src/guidellm/scheduler/strategy.py @@ -10,8 +10,8 @@ from pydantic import Field -from guidellm.config import settings from guidellm.objects import StandardBaseModel +from guidellm.settings import settings __all__ = [ "AsyncConstantStrategy", diff --git a/src/guidellm/config.py b/src/guidellm/settings.py similarity index 100% rename from src/guidellm/config.py rename to src/guidellm/settings.py diff --git a/src/guidellm/utils/text.py b/src/guidellm/utils/text.py index 539ea8a0..3b9a2e26 100644 --- a/src/guidellm/utils/text.py +++ b/src/guidellm/utils/text.py @@ -10,7 +10,7 @@ from loguru import logger from guidellm import data as package_data -from guidellm.config import settings +from guidellm.settings import settings __all__ = [ "EndlessTextCreator", diff --git a/tests/unit/backend/test_openai_backend.py b/tests/unit/backend/test_openai_backend.py index 0a4c2c38..7123c590 100644 --- a/tests/unit/backend/test_openai_backend.py +++ b/tests/unit/backend/test_openai_backend.py @@ -3,7 +3,7 @@ import pytest from guidellm.backend import OpenAIHTTPBackend, ResponseSummary, StreamingTextResponse -from guidellm.config import settings +from guidellm.settings import settings @pytest.mark.smoke diff --git a/tests/unit/backend/test_openai_backend_custom_configs.py b/tests/unit/backend/test_openai_backend_custom_configs.py index 7f6706ad..5855152d 100644 --- a/tests/unit/backend/test_openai_backend_custom_configs.py +++ b/tests/unit/backend/test_openai_backend_custom_configs.py @@ -1,7 +1,7 @@ import pytest from guidellm.backend import OpenAIHTTPBackend -from guidellm.config import settings +from guidellm.settings import settings @pytest.mark.smoke diff --git a/tests/unit/presentation/test_injector.py b/tests/unit/presentation/test_injector.py index b2ff7116..da269815 100644 --- a/tests/unit/presentation/test_injector.py +++ b/tests/unit/presentation/test_injector.py @@ -3,8 +3,8 @@ import pytest from pydantic import BaseModel -from guidellm.config import settings from guidellm.presentation.injector import create_report, inject_data +from guidellm.settings import settings class ExampleModel(BaseModel): diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index f5d9415c..42c8901d 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -1,6 +1,6 @@ import pytest -from guidellm.config import ( +from guidellm.settings import ( DatasetSettings, Environment, LoggingSettings, diff --git a/tests/unit/test_logger.py b/tests/unit/test_logger.py index 53e8b664..792c9770 100644 --- a/tests/unit/test_logger.py +++ b/tests/unit/test_logger.py @@ -3,7 +3,7 @@ import pytest from guidellm import configure_logger, logger -from guidellm.config import LoggingSettings +from guidellm.settings import LoggingSettings @pytest.fixture(autouse=True)