Move to uv and pyproject.toml (#23)

* Move to uv and pyproject.toml

Signed-off-by: Hemil Desai <[email protected]>

* Dependency update (#35)

* update required deps

Signed-off-by: Maanu Grover <[email protected]>

* add optional groups for docs and test

Signed-off-by: Maanu Grover <[email protected]>

* install deps from uv in ci

Signed-off-by: Maanu Grover <[email protected]>

* try again

Signed-off-by: Maanu Grover <[email protected]>

* update ci dockerfile

Signed-off-by: Maanu Grover <[email protected]>

---------

Signed-off-by: Maanu Grover <[email protected]>

---------

Signed-off-by: Hemil Desai <[email protected]>
Signed-off-by: Maanu Grover <[email protected]>
Co-authored-by: Maanu Grover <[email protected]>

Author: hemildesai

.github/workflows/code-linting.yml

@@ -25,9 +25,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.7.10"
+
       - name: Install dependencies
         run: |
-          pip install -r requirements/requirements_test.txt
+          uv sync --only-group test
 
       - name: Get changed files
         id: changed-files
@@ -60,7 +65,7 @@ jobs:
           fi
 
           set +e
-          pylint $ADDITIONAL_PYLINT_ARGS --output "pylintrc.txt" --rcfile ".pylintrc" ${CHANGED_FILES[@]}
+          uv run --only-group test pylint $ADDITIONAL_PYLINT_ARGS --output "pylintrc.txt" --rcfile ".pylintrc" ${CHANGED_FILES[@]}
           echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
 
       - name: Run flake8
@@ -82,7 +87,7 @@ jobs:
           fi
 
           set +e
-          flake8 $ADDITIONAL_FLAKE8_ARGS --output "flake8.txt" --config ".flake8" ${CHANGED_FILES[@]}
+          uv run --only-group test flake8 $ADDITIONAL_FLAKE8_ARGS --output "flake8.txt" --config ".flake8" ${CHANGED_FILES[@]}
           echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT"
 
       - name: Run isort
@@ -98,7 +103,7 @@ jobs:
           fi
 
           set +e
-          isort --check-only --diff ${CHANGED_FILES[@]} > isort.txt
+          uv run --only-group test isort --check-only --diff ${CHANGED_FILES[@]} > isort.txt
           ISORT_EXIT_CODE=$?
 
           if [[ $SKIP_LINTING == true ]]; then
@@ -120,7 +125,7 @@ jobs:
           fi
 
           set +e
-          black --check --diff ${CHANGED_FILES[@]} > black.txt
+          uv run --only-group test black --check --diff ${CHANGED_FILES[@]} > black.txt
           BLACK_EXIT_CODE=$?
 
           if [[ $SKIP_LINTING == true ]]; then

docker/Dockerfile.ci

@@ -17,11 +17,10 @@ ENV PIP_CONSTRAINT=""
 
 # Install all
 WORKDIR /workspace
-RUN --mount=type=bind,source=setup.py,target=/workspace/setup.py \
+RUN --mount=type=bind,source=pyproject.toml,target=/workspace/pyproject.toml \
     --mount=type=bind,source=nemo_lm/__init__.py,target=/workspace/nemo_lm/__init__.py \
     --mount=type=bind,source=nemo_lm/package_info.py,target=/workspace/nemo_lm/package_info.py \
     --mount=type=bind,source=README.md,target=/workspace/README.md \
-    --mount=type=bind,source=requirements,target=/workspace/requirements \
     pip install --no-build-isolation --no-cache-dir '.[all]'
 
 # Install development versions

pyproject.toml

@@ -12,13 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-[build-system]
-requires = ["setuptools >= 61.0"]
-build-backend = "setuptools.build_meta"
-
 [project]
 name = "nemo-lm"
-dynamic = ["dependencies", "optional-dependencies", "version"]
+dynamic = ["version"]
 description = "NeMo - a toolkit for Conversational AI"
 readme = "README.md"
 license = {file = "LICENSE"}
@@ -59,6 +55,54 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "Topic :: Utilities",
 ]
+dependencies = [
+    "accelerate>=1.6.0",
+    "datasets>=3.5.1",
+    "numpy>=2.2.5",
+    "omegaconf>=2.3.0",
+    "packaging>=25.0",
+    "tensorboard>=2.19.0",
+    "torch>=2.7.0",
+    "transformers>=4.51.3",
+    "wandb>=0.19.10",
+    "megatron-core",
+    "six>=1.17.0",
+    "regex>=2024.11.6",
+    "pyyaml>=6.0.2",
+    "einops>=0.8.1",
+    "sentencepiece>=0.2.0",
+    "tiktoken>=0.9.0",
+    "tqdm>=4.67.1",
+    "nvidia-modelopt[torch]>=0.25.0,<=0.29.0 ; sys_platform != 'darwin'",
+    "nvidia-resiliency-ext>=0.3.0,<1.0.0 ; sys_platform != 'darwin'",
+]
+
+[tool.uv.sources]
+megatron-core = { git = "https://github.com/NVIDIA/Megatron-LM.git" }
+
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[dependency-groups]
+docs = [
+    "myst-parser>=4.0.1",
+    "nvidia-sphinx-theme>=0.0.8",
+    "sphinx>=8.1.3",
+    "sphinx-autobuild>=2024.10.3",
+    "sphinx-autodoc2>=0.5.0",
+    "sphinx-copybutton>=0.5.2",
+]
+test = [
+    "black>=25.1.0",
+    "coverage>=7.8.1",
+    "flake8>=7.2.0",
+    "isort>=6.0.1",
+    "pylint>=3.3.7",
+    "pytest>=8.3.5",
+    "pytest-mock>=3.14.0",
+    "pytest-runner>=6.0.1",
+]
 
 [tool.setuptools]
 py-modules = ["nemo_lm"]
@@ -81,7 +125,6 @@ sections = ["FUTURE", "STDLIB", "THIRDPARTY", "FIRSTPARTY", "LOCALFOLDER"]
 default_section = "THIRDPARTY"
 extend_skip = ["setup.py", "docs/source/conf.py"]
 
-
 [tool.black]
 line_length = 119
 skip_string_normalization = true