Nemotron/pyproject.toml at main · NVIDIA-NeMo/Nemotron · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
[project]
name = "nemotron"
version = "0.1.0"
description = "Reproducible training recipes for NVIDIA Nemotron model family - transparent pipelines for data preparation, training, and evaluation across all stages"
requires-python = ">=3.10"
license = {text = "MIT"}
authors = [
    {name = "Nemotron Contributors"}
]
keywords = ["nemotron", "nvidia", "llm", "training", "reproducible", "pipelines", "machine-learning"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Developers",
    "Intended Audience :: Science/Research",
    "Intended Audience :: Education",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]

dependencies = [
    "nemo-run>=0.4.0",
    "pydantic>=2.0.0",
    "typer>=0.12.0",
    "omegaconf>=2.3.0",
    "rich>=13.0.0",
    "textual>=0.70.0",
    "ray[default]==2.49.2",
    "fsspec>=2024.0.0",
    "numpy>=1.24.0",
    "pyarrow>=14.0.0",
    "xxhash>=3.4.0",
    "transformers>=4.36.0",
    "huggingface_hub>=0.20.0",
    "datasets>=2.14.0",  # Required for ray.data.from_huggingface
    "pyyaml>=6.0",
    "tomli>=2.0.0;python_version<'3.11'",
    "tomlkit>=0.12.0",
    "colorama>=0.4.6",
    "wandb>=0.23.1",
    "jinja2>=3.0.0",  # Required for chat templates in transformers
    "pydantic-settings>=2.12.0",
]

[project.optional-dependencies]
wandb = ["wandb>=0.15.0"]
s3 = ["s3fs>=2024.0.0"]
gcs = ["gcsfs>=2024.0.0"]
sentencepiece = ["sentencepiece>=0.2.0"]
xenna = ["cosmos-xenna"]
dev = [
    "pytest>=7.0.0",
    "pytest-cov>=4.0.0",
    "mypy>=1.0.0",
    "ruff>=0.1.0",
]
all = [
    "wandb>=0.15.0",
    "s3fs>=2024.0.0",
    "gcsfs>=2024.0.0",
    "sentencepiece>=0.2.0",
    "cosmos-xenna",
]

# Note: megatron-bridge is required for training but not listed as a dependency
# because it requires CUDA/nvcc to build. Install separately in GPU environments:
#   pip install megatron-bridge>=0.1.0

[project.urls]
Homepage = "https://github.com/nemotron/nemotron"
Documentation = "https://github.com/nemotron/nemotron#readme"
Repository = "https://github.com/nemotron/nemotron"

[project.scripts]
nemotron = "nemotron.__main__:main"

[project.entry-points."fsspec.specs"]
art = "nemo_runspec.filesystem:ArtifactFileSystem"

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.hatch.build.targets.wheel]
packages = ["src/nemotron", "src/nemo_runspec"]

[tool.pytest.ini_options]
testpaths = ["tests"]
python_files = ["test_*.py"]
python_functions = ["test_*"]
markers = [
    "integration: End-to-end tests that run real stage code with synthetic data",
    "gpu: Tests that require NVIDIA GPU and nemo_automodel",
    "docker: Tests that require Docker with NVIDIA runtime",
]

[tool.mypy]
python_version = "3.10"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true

[tool.ruff]
line-length = 119
target-version = "py310"
exclude = [
    "usage-cookbook",
    "use-case-examples",
]

[tool.ruff.lint]
select = ["E", "F", "I", "N", "W", "UP"]
ignore = []

[tool.uv]
package = true

[dependency-groups]
dev = [
    "pytest>=9.0.2",
]
run = [
    "nemo-run>=0.4.0",
]
docs = [
    "myst-parser>=4.0.1",
    "nvidia-sphinx-theme>=0.0.8",
    "sphinx>=8.1.3",
    "sphinx-autobuild>=2024.10.3",
    "sphinx-autodoc2>=0.5.0",
    "sphinx-copybutton>=0.5.2",
    "sphinx-design>=0.6.0",
    "sphinxcontrib-mermaid",
]