pyproject.toml

# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[build-system]
requires = ["setuptools>=42", "wheel", "cython>=3.0.0"]
build-backend = "setuptools.build_meta"

[tool.setuptools]
package-dir = { "" = "src" }

[tool.setuptools.packages.find]
where = ["src"]
include = ["nemo_eval", "nemo_eval.*"]

[tool.setuptools.dynamic]
version = { attr = "nemo_eval.__version__" }
readme = { file = "README.md", content-type = "text/markdown" }

[project]
name = "nemo-eval"
dynamic = ["version", "readme"]
description = "NeMo Eval: Evaluation Utilities for LLM and VLM models"
license = { file = "LICENSE" }
requires-python = ">=3.10"
authors = [{ name = "NVIDIA", email = "nemo-toolkit@nvidia.com" }]
maintainers = [{ name = "NVIDIA", email = "nemo-toolkit@nvidia.com" }]
keywords = [
    "NLP",
    "NeMo",
    "deep",
    "gpu",
    "language",
    "learning",
    "machine",
    "nvidia",
    "pytorch",
    "torch",
]
classifiers = [
    "Development Status :: 5 - Production/Stable",
    "Environment :: Console",
    "Intended Audience :: Developers",
    "Intended Audience :: Information Technology",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Apache Software License",
    "Natural Language :: English",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Scientific/Engineering :: Image Recognition",
    "Topic :: Scientific/Engineering :: Mathematics",
    "Topic :: Scientific/Engineering",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: Software Development :: Libraries",
    "Topic :: Utilities",
]
dependencies = [
    "nvidia-lm-eval==25.6.1",
    "uvicorn",
    "flask",
    "megatron-core>=0.13.0a0,<0.14.0",
    "nvidia-modelopt[torch,onnx]>=0.31.0a0,<0.32.0; sys_platform != 'darwin'",
    "nvidia-resiliency-ext>=0.3.0a0,<0.4.0; sys_platform != 'darwin'",
    "nemo-export-deploy>=0.1.0a0,<0.2.0",
    "pandas>2.0.0",
]

[project.optional-dependencies]
te = ["transformer-engine[pytorch]==2.5.0"]

[project.urls]
Download = "https://github.com/NVIDIA-NeMo/Eval/releases"
Homepage = "https://github.com/NVIDIA-NeMo/Eval"

[dependency-groups]
# This is a default group so that we install these even with bare `uv sync`
build = ["setuptools", "torch", "pybind11", "Cython>=3.0.0"]
docs = [
    "sphinx",
    "sphinx-autobuild",    # For live doc serving while editing docs
    "sphinx-autodoc2",     # For documenting Python API
    "sphinx-copybutton",   # Adds a copy button for code blocks
    "myst_parser",         # For our markdown docs
    "nvidia-sphinx-theme", # Our NVIDIA theme
]
linting = ["ruff~=0.9.0"]
test = ["pytest", "pytest-mock", "coverage", "click", "pytest-httpserver"]
nemo-toolkit = [
    "nemo-toolkit[automodel,common-only,nlp-only,eval,multimodal-only]~=2.4.0",
]
nemo-run = ["nemo-run"]

# # uv.sources allows us to override dependencies with VCS commits. 
# # Lets use this only for debugging purposes, but not for production (main).
# [tool.uv.sources]
# nemo-toolkit = { git = "https://github.com/NVIDIA/NeMo.git", rev = "5e63ca3c1631bb156187705f9c4cf643a67f9c05" }

[tool.uv]
# Currently, TE must be built with no build-isolation b/c it requires torch
no-build-isolation-package = ["transformer-engine", "transformer-engine-torch"]
# Always apply the build group since dependencies like TE/mcore/nemo-run require build dependencies
# and this lets us assume they are implicitly installed with a simply `uv sync`. Ideally, we'd
# avoid including these in the default dependency set, but for now it's required.
default-groups = ["linting", "build", "test", "nemo-run"]
# Users may use different link-modes depending on their scenario:
#  --link-mode=hardlink (default on linux; may get warnings about switching to --link-mode copy if uv cache and venv on different file-systems)
#  --link-mode=copy (slower but more reliable; supresses warning)
#  --link-mode=symlink (fastest option when uv cache and venv on different file-system; caveat: venv is brittle since it depends on the environment/container)
link-mode = "copy"
override-dependencies = [
    "urllib3>1.27.0",
    "tiktoken>=0.9.0",  # because nemo-toolkit and megatron-bridge disagree on tiktoken, we need to pin it here,
    "protobuf>=4.25.1",
    "pandas>2.0.0",
]
prerelease = "allow"

[[tool.uv.index]]
name = "pypi"
url = "https://pypi.org/simple"
explicit = true

[[tool.uv.index]]
name = "pytorch-cu128"
url = "https://download.pytorch.org/whl/cu128"
explicit = true

[tool.pytest.ini_options]
addopts = "--durations=15 -s -rA -x"
testpaths = ["tests"]
python_files = "test_*.py"

[tool.coverage.run]
concurrency = ["thread", "multiprocessing"]
omit = [
    "/tmp/*",
    "/workspace/tests/*",
    "/usr/local/lib/python3.12/*",
    "*_ray.py",
    "ray_*.py",
    "/workspace/*.py",
    "*package_info.py",
]
parallel = true
sigterm = true

[tool.coverage.paths]
source = [".", "/workspace", "/home/runner/work/Eval/Eval"]

[tool.ruff]
line-length = 120

[tool.ruff.format]
quote-style = "double"
skip-magic-trailing-comma = false
line-ending = "auto"

[tool.ruff.lint]
# Enable all `pydocstyle` rules, limiting to those that adhere to the
# Google convention via `convention = "google"`, below.
select = [
    "F541", # f-string without any placeholders
    "F841", # local variable assigned but never used
    "F401", # imported but unused
    "E741", # ambiguous variable name
    "F821", # undefined name
    "E266", # too many leading '#' for block comment
    "I",    # isort
    "D101", # docstring
    "D103",
]

ignore = [
    "E501", # Line too long - handled by formatter
    "D101",
    "D103",
]

[tool.ruff.lint.pydocstyle]
convention = "google"

# Section to exclude errors for different file types
[tool.ruff.per-file-ignores]
# Ignore all directories named `tests`.
"tests/**" = ["D"]
# Ignore all files that end in `_test.py`.
"*_test.py" = ["D"]
# Ignore F401 (import but unused) in __init__.py
"__init__.py" = ["F401"]