ci: Add build-test-publish-wheel workflow (#114)

Signed-off-by: oliver könig <[email protected]>

Author: ko3n1g

.github/workflows/build-test-publish-wheel.yml

@@ -0,0 +1,41 @@
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build, test, and publish a PyPi wheel (to testpypi)
+
+on:
+  push:
+    branches:
+      - "main"
+      - "[rv][0-9].[0-9].[0-9]"
+      - "[rv][0-9].[0-9].[0-9]rc[0-9]"
+  pull_request:
+
+defaults:
+  run:
+    shell: bash -x -e -u -o pipefail {0}
+
+jobs:
+  build-test-publish-wheel:
+    uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/[email protected]
+    with:
+      dry-run: true
+      python-package: nemo_run
+      python-version: "3.10"
+      packaging: hatch
+    secrets:
+      TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
+      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
+      SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
+      SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}

nemo_run/__init__.py

@@ -13,14 +13,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from nemo_run import cli
 from nemo_run.api import autoconvert, dryrun_fn
 from nemo_run.config import Config, ConfigurableMixin, Partial, Script
-from nemo_run.core.execution.base import (
-    Executor,
-    ExecutorMacros,
-    import_executor,
-)
+from nemo_run.core.execution.base import Executor, ExecutorMacros, import_executor
 from nemo_run.core.execution.dgxcloud import DGXCloudExecutor
 from nemo_run.core.execution.docker import DockerExecutor
 from nemo_run.core.execution.launcher import FaultTolerance, SlurmRay, SlurmTemplate, Torchrun
@@ -32,10 +27,13 @@
 from nemo_run.devspace.base import DevSpace
 from nemo_run.help import help
 from nemo_run.lazy import LazyEntrypoint, lazy_imports
+from nemo_run.package_info import __package_name__, __version__
 from nemo_run.run.api import run
 from nemo_run.run.experiment import Experiment
 from nemo_run.run.plugin import ExperimentPlugin as Plugin
 
+from nemo_run import cli
+
 __all__ = [
     "autoconvert",
     "cli",
@@ -70,9 +68,6 @@
     "Torchrun",
     "SlurmRay",
     "SlurmTemplate",
+    "__version__",
+    "__package_name__",
 ]
-
-try:
-    from nemo_run._version import __version__
-except Exception:
-    __version__ = "0.0.1"

nemo_run/package_info.py

@@ -0,0 +1,38 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from packaging.version import Version
+
+__version__ = "0.2.0rc0.dev0"
+
+MAJOR = Version(__version__).major
+MINOR = Version(__version__).minor
+PATCH = 3082
+if pre := Version(__version__).pre:
+    PRE_RELEASE = "".join(map(str, pre))
+else:
+    PRE_RELEASE = ""
+
+DEV = Version(__version__).dev
+
+__package_name__ = "nemo_curator"
+__contact_names__ = "NVIDIA"
+__contact_emails__ = "[email protected]"
+__homepage__ = "https://docs.nvidia.com/nemo-framework/user-guide/latest/datacuration/index.html"
+__repository_url__ = "https://github.com/NVIDIA/NeMo-Curator"
+__download_url__ = "https://github.com/NVIDIA/NeMo-Curator/releases"
+__description__ = (
+    "NeMo Curator - Scalable Data Preprocessing Tool for Training Large Language Models"
+)
+__license__ = "Apache2"
+__keywords__ = "deep learning, machine learning, gpu, NLP, NeMo, nvidia, pytorch, torch, language, preprocessing, LLM, large language model"

pyproject.toml

@@ -29,7 +29,8 @@ dependencies = [
     "jinja2>=3.1.4",
     "cryptography < 43.0.0",
     "networkx >= 3.3",
-    "omegaconf>=2.3.0"
+    "omegaconf>=2.3.0",
+    "packaging"
 ]
 readme = "README.md"
 requires-python = ">= 3.10"
@@ -125,10 +126,7 @@ allow-direct-references = true
 packages = ["nemo_run"]
 
 [tool.hatch.version]
-source = "vcs"
-
-[tool.hatch.build.hooks.vcs]
-version-file = "nemo_run/_version.py"
+path  = "nemo_run/package_info.py"
 
 [tool.ruff]
 line-length = 100