Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .github/workflows/build-docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,29 @@ on:
branches:
- main
- "pull-request/[0-9]+"
- "deploy-release/*"

jobs:
pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.53.0

build-docs:
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_docs.yml@v0.48.0

build-docs-summary:
needs: [pre-flight, build-docs]
if: |
(
needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| success()
)
&& !cancelled()
runs-on: ubuntu-latest
steps:
- name: Result
run: echo Build docs successful
29 changes: 27 additions & 2 deletions .github/workflows/build-test-publish-wheel.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,47 @@ on:
branches:
- main
- "r**"
- "pull-request/**"
- "deploy-release/*"

defaults:
run:
shell: bash -x -e -u -o pipefail {0}

jobs:
pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.53.0

build-test-publish-wheel:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.33.0
if: ${{ vars.BUILD_TEST_PUBLISH_WHEEL == 'true' }}
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_build_test_publish_wheel.yml@v0.48.0
with:
dry-run: true
python-package: nemo_eval
python-version: "3.10"
packaging: uv
has-src-dir: true
no-publish: ${{ !(github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/r')) }}
secrets:
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ENDPOINT }}
SLACK_WEBHOOK_ADMIN: ${{ secrets.SLACK_WEBHOOK_ADMIN }}
GH_TOKEN: ${{ secrets.PAT }}

build-test-publish-wheel-summary:
needs: [pre-flight, build-test-publish-wheel]
if: |
(
needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| success()
)
&& !cancelled()
runs-on: ubuntu-latest
steps:
- name: Result
run: echo Build test publish wheel successful
94 changes: 41 additions & 53 deletions .github/workflows/cicd-main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ on:
branches:
- main
- "pull-request/[0-9]+"
- "deploy-release/*"

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }}
Expand All @@ -30,73 +31,39 @@ permissions:

jobs:
pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.53.0

linting:
runs-on: ubuntu-latest
outputs:
is_ci_workload: ${{ steps.is_ci_workload.outputs.main }}
force_run_all: ${{ steps.force_run_all.outputs.main }}
docs_only: ${{ steps.docs_only.outputs.main == 'true' }}
env:
TESTS_TO_RUN: ${{ inputs.test_to_run }}
EVENT_NAME: ${{ github.event_name }}
HAS_LABEL: ${{ github.event.label.name == 'Run CICD' }}
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.is_ci_workload == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| needs.pre-flight.outputs.docs_only == 'true')
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Get changed files
id: changed-files
uses: step-security/changed-files@v45.0.1
- name: Setup Python
uses: actions/setup-python@v5
with:
files_yaml: |
doc:
- '**.md'
- docs/**
src:
- '!**.md'
- '!docs/**'

- name: Check if docs only
shell: bash
id: docs_only
env:
DOCS_ONLY: ${{ steps.changed-files.outputs.doc_any_changed == 'true' && steps.changed-files.outputs.src_any_changed == 'false' }}
run: |
echo "main=$DOCS_ONLY" | tee -a "$GITHUB_OUTPUT"

- name: Check if this is a CI workload
shell: bash
id: is_ci_workload
python-version: "3.10"
- name: Install ruff
run: |
branch_name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}

if [[ "$branch_name" =~ ^bump-ci-container || "$EVENT_NAME" == "schedule" ]]; then
is_ci_workload=true
else
is_ci_workload=false
fi

echo "main=$is_ci_workload" | tee -a "$GITHUB_OUTPUT"
pip install ruff

- name: Check if force-run-all is set
shell: bash
id: force_run_all
env:
HAS_FORCE_RUN_ALL_LABEL: ${{ contains(github.event.pull_request.labels.*.name, 'force-run-all') }}
- name: Check lint
run: |
if [[ "$HAS_FORCE_RUN_ALL_LABEL" == "true" || "$EVENT_NAME" == "schedule" ]]; then
force_run_all=true
else
force_run_all=false
fi

echo "main=$force_run_all" | tee -a "$GITHUB_OUTPUT"
pip install pre-commit==3.6.0
pre-commit install
pre-commit run --all-files --show-diff-on-failure --color=always

cicd-wait-in-queue:
runs-on: ubuntu-latest
needs: [pre-flight]
needs: [pre-flight, linting]
environment: test
if: |
!(needs.pre-flight.outputs.is_ci_workload == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| needs.pre-flight.outputs.docs_only == 'true')
steps:
- name: Running CI tests
Expand Down Expand Up @@ -185,6 +152,7 @@ jobs:
if: |
(
needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| success()
)
&& !cancelled()
Expand Down Expand Up @@ -280,9 +248,29 @@ jobs:
exit 1
fi

coverage-placeholder:
needs: [pre-flight, Nemo_CICD_Test]
name: codecov/patch
if: |
(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
&& !failure()
&& !cancelled()
runs-on: ubuntu-latest
steps:
- name: Dummy placeholder
run: echo "noop"

Coverage:
runs-on: ubuntu-latest
needs: [Nemo_CICD_Test]
if: |
(
needs.pre-flight.outputs.docs_only == 'false'
&& needs.pre-flight.outputs.is_deployment_workflow == 'false'
&& success()
)
&& !cancelled()
strategy:
matrix:
flag: [unit-test, e2e]
Expand Down
35 changes: 0 additions & 35 deletions .github/workflows/code-linting.yml

This file was deleted.

27 changes: 26 additions & 1 deletion .github/workflows/copyright-check.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,33 @@
name: Copyright check

on:
pull_request:
push:
branches:
- main
- "pull-request/[0-9]+"
- "deploy-release/*"

jobs:
pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.53.0

copyright-check:
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_copyright_check.yml@v0.2.0

copyright-check-summary:
needs: [pre-flight, copyright-check]
if: |
(
needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| success()
)
&& !cancelled()
runs-on: ubuntu-latest
steps:
- name: Result
run: echo Copyright check successful
29 changes: 28 additions & 1 deletion .github/workflows/install-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,21 @@ on:
branches:
- main
- "pull-request/[0-9]+"
- "deploy-release/*"
env:
UV_PROJECT_ENVIRONMENT: "./venv"

jobs:
pre-flight:
uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.53.0

pip-test-no-cuda:
runs-on: ${{ matrix.arch }}
name: Pip - Python${{ matrix.python-version }} - ${{ matrix.arch == 'ubuntu-latest' && 'AMD64/Linux' || (matrix.arch == 'ubuntu-24.04-arm' && 'ARM64/Linux' || 'ARM64/Darwin') }} - No CUDA
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -80,6 +88,10 @@ jobs:
pip-test-cuda:
runs-on: linux-amd64-cpu16
name: Pip - Python${{ matrix.python-version }}${{ matrix.extra-groups != '' && format('[{0}]', matrix.extra-groups) || '' }} - AMD64/Linux - NGC CUDA
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
container:
image: nvcr.io/nvidia/cuda:12.8.1-cudnn-devel-ubuntu24.04
environment: nemo-ci
Expand Down Expand Up @@ -134,6 +146,10 @@ jobs:
uv-test-cuda:
runs-on: linux-amd64-cpu16
name: UV - AMD64/Linux - NGC ${{ contains(matrix.image, 'cuda') && 'CUDA' || 'PyTorch' }}
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
container:
image: ${{ matrix.image }}
environment: nemo-ci
Expand Down Expand Up @@ -185,6 +201,10 @@ jobs:
runs-on: ${{ matrix.arch }}
name: UV - Python ${{ matrix.python-version }} - ${{ matrix.arch == 'ubuntu-latest' && 'AMD64/Linux' || 'ARM64/Darwin' }} - No CUDA
environment: nemo-ci
needs: [pre-flight]
if: |
!(needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true')
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -234,9 +254,16 @@ jobs:
python-binary: ./venv/bin/python

install-test-summary:
needs: [pip-test-no-cuda, uv-test-cuda, pip-test-cuda, uv-test-no-cuda]
needs: [pre-flight, pip-test-no-cuda, uv-test-cuda, pip-test-cuda, uv-test-no-cuda]
runs-on: ubuntu-latest
name: Install test summary
if: |
(
needs.pre-flight.outputs.docs_only == 'true'
|| needs.pre-flight.outputs.is_deployment_workflow == 'true'
|| success()
)
&& !cancelled()
steps:
- name: Result
run: echo Install check successful
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -104,9 +104,6 @@ target/
profile_default/
ipython_config.py

# pyenv
.python-version

# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
Expand Down
1 change: 1 addition & 0 deletions .python-version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3.12
Loading