Update release-docs workflow to use FW-CI-templates v0.72.0 #2445
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: CICD NeMo | |
| on: | |
| schedule: | |
| - cron: 0 0 * * * | |
| push: | |
| branches: | |
| - main | |
| - "pull-request/[0-9]+" | |
| - "deploy-release/*" | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.label.name || 'main' }}-${{ github.event_name }} | |
| cancel-in-progress: true | |
| permissions: | |
| id-token: write | |
| contents: read | |
| jobs: | |
| pre-flight: | |
| uses: NVIDIA-NeMo/FW-CI-templates/.github/workflows/_cicd_preflight.yml@v0.69.1 | |
| with: | |
| default_runner_prefix: ${{ vars.DEFAULT_RUNNER_PREFIX }} | |
| non_nvidia_runner_prefix: ${{ vars.NON_NVIDIA_RUNNER_PREFIX }} | |
| default_test_data_path: ${{ vars.DEFAULT_TEST_DATA_PATH }} | |
| non_nvidia_test_data_path: ${{ vars.NON_NVIDIA_TEST_DATA_PATH }} | |
| secrets: | |
| NVIDIA_MANAGEMENT_ORG_PAT: ${{ secrets.NVIDIA_MANAGEMENT_ORG_PAT }} | |
| linting: | |
| runs-on: ubuntu-latest | |
| needs: [pre-flight] | |
| if: | | |
| ( | |
| needs.pre-flight.outputs.is_deployment_workflow == 'false' | |
| && needs.pre-flight.outputs.is_ci_workload == 'true' | |
| ) || ( | |
| needs.pre-flight.outputs.is_deployment_workflow == 'false' | |
| && needs.pre-flight.outputs.is_ci_workload == 'false' | |
| && needs.pre-flight.outputs.docs_only == 'false' | |
| ) | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.10" | |
| - name: Install ruff | |
| run: | | |
| pip install ruff | |
| - name: Run ruff | |
| run: | | |
| ruff check . | |
| ruff format --check . | |
| cicd-wait-in-queue: | |
| needs: [pre-flight, linting] | |
| runs-on: ubuntu-latest | |
| environment: test | |
| if: | | |
| !(needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.is_deployment_workflow == 'true' | |
| || needs.pre-flight.outputs.docs_only == 'true') | |
| steps: | |
| - name: Running CI tests | |
| run: | | |
| echo "Running CI tests" | |
| cicd-unit-tests-trtllm: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - script: L0_Unit_Tests_GPU | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| - script: L0_Unit_Tests_CPU | |
| runner: linux-amd64-cpu16 | |
| cpu-only: true | |
| needs: [pre-flight, cicd-wait-in-queue] | |
| if: | | |
| ( | |
| success() | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.force_run_all == 'true' | |
| ) | |
| && !cancelled() | |
| runs-on: ${{ matrix.runner }} | |
| name: ${{ matrix.script }} | |
| environment: nemo-ci | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: ${{ matrix.script }} | |
| timeout: 60 | |
| is_unit_test: "true" | |
| cpu-only: ${{ matrix.cpu-only || false }} | |
| has-azure-credentials: "true" | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: trtllm | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ matrix.runner }} | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| cicd-unit-tests-vllm: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - script: L0_Unit_Tests_GPU | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| - script: L0_Unit_Tests_CPU | |
| runner: linux-amd64-cpu16 | |
| cpu-only: true | |
| needs: [pre-flight, cicd-wait-in-queue] | |
| if: | | |
| ( | |
| success() | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.force_run_all == 'true' | |
| ) | |
| && !cancelled() | |
| runs-on: ${{ matrix.runner }} | |
| name: ${{ matrix.script }} | |
| environment: nemo-ci | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: ${{ matrix.script }} | |
| timeout: 60 | |
| is_unit_test: "true" | |
| cpu-only: ${{ matrix.cpu-only || false }} | |
| has-azure-credentials: "true" | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: vllm | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ matrix.runner }} | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| cicd-e2e-tests-trtllm: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - script: L2_Launch_TRTLLM | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| - script: L2_TRTLLM_API_Deploy_Query | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| needs: [pre-flight, cicd-unit-tests-trtllm] | |
| runs-on: ${{ matrix.runner }} | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| if: | | |
| ( | |
| success() | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.force_run_all == 'true' | |
| ) | |
| && !cancelled() | |
| environment: nemo-ci | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: ${{ matrix.script }} | |
| is_optional: ${{ matrix.is_optional || false }} | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| has-azure-credentials: true | |
| is_unit_test: "false" | |
| timeout: 60 | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: trtllm | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ matrix.runner }} | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| cicd-e2e-tests-trt-onnx: | |
| needs: [cicd-unit-tests-trtllm, pre-flight] | |
| runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| environment: nemo-ci | |
| if: | | |
| ( | |
| success() | |
| || ( | |
| needs.cicd-wait-in-queue.result == 'skipped' | |
| && needs.pre-flight.outputs.is_ci_workload == 'true' | |
| ) | |
| ) | |
| && !cancelled() | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: L2_ONNX_TRT | |
| is_optional: ${{ matrix.is_optional || false }} | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| has-azure-credentials: true | |
| is_unit_test: "false" | |
| timeout: 60 | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: trt-onnx | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| cicd-e2e-tests-vllm: | |
| needs: [cicd-unit-tests-vllm, pre-flight] | |
| runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| environment: nemo-ci | |
| if: | | |
| ( | |
| success() | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.force_run_all == 'true' | |
| ) | |
| && !cancelled() | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: L2_Launch_vLLM | |
| is_optional: ${{ matrix.is_optional || false }} | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| has-azure-credentials: true | |
| is_unit_test: "false" | |
| timeout: 60 | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: vllm | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| cicd-e2e-tests-inframework: | |
| needs: [pre-flight, cicd-unit-tests-trtllm, cicd-unit-tests-vllm] | |
| runs-on: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| name: ${{ matrix.is_optional && 'PLEASEFIXME_' || '' }}${{ matrix.script }} | |
| environment: nemo-ci | |
| if: | | |
| ( | |
| success() | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || needs.pre-flight.outputs.force_run_all == 'true' | |
| ) | |
| && !cancelled() | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: main | |
| uses: ./.github/actions/test-template | |
| with: | |
| script: L2_Launch_InFramework | |
| is_optional: ${{ matrix.is_optional || false }} | |
| azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| has-azure-credentials: true | |
| is_unit_test: "false" | |
| timeout: 60 | |
| PAT: ${{ secrets.PAT }} | |
| inference-framework: inframework | |
| test-data-path: ${{ needs.pre-flight.outputs.test_data_path }} | |
| runner: ${{ needs.pre-flight.outputs.runner_prefix }}-gpu-x2 | |
| ngc-api-user: ${{ secrets.NGC_API_USER }} | |
| ngc-api-key: ${{ secrets.NGC_API_KEY }} | |
| Nemo_CICD_Test: | |
| needs: | |
| - pre-flight | |
| - cicd-unit-tests-trtllm | |
| - cicd-unit-tests-vllm | |
| - cicd-e2e-tests-trtllm | |
| - cicd-e2e-tests-vllm | |
| - cicd-e2e-tests-inframework | |
| if: | | |
| ( | |
| needs.pre-flight.outputs.docs_only == 'true' | |
| || needs.pre-flight.outputs.is_deployment_workflow == 'true' | |
| || needs.pre-flight.outputs.is_ci_workload == 'true' | |
| || always() | |
| ) | |
| && !cancelled() | |
| runs-on: ubuntu-latest | |
| permissions: write-all | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Get workflow result | |
| id: result | |
| shell: bash -x -e -u -o pipefail {0} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| RUN_ID: ${{ github.run_id }} | |
| SKIPPING_IS_ALLOWED: ${{ needs.pre-flight.outputs.docs_only == 'true' || needs.pre-flight.outputs.is_deployment_workflow == 'true' || needs.pre-flight.outputs.is_ci_workload == 'true' }} | |
| run: | | |
| FAILED_JOBS=$(gh run view $GITHUB_RUN_ID --json jobs --jq '[.jobs[] | select(.status == "completed" and .conclusion != "success")] | length') || echo 0 | |
| if [ "${FAILED_JOBS:-0}" -eq 0 ] || [ "$SKIPPING_IS_ALLOWED" == "true" ]; then | |
| echo "✅ All previous jobs completed successfully" | |
| exit 0 | |
| else | |
| echo "❌ Found $FAILED_JOBS failed job(s)" | |
| # Show which jobs failed | |
| gh run view $GITHUB_RUN_ID --json jobs --jq '.jobs[] | select(.status == "completed" and .conclusion != "success") | .name' | |
| exit 1 | |
| fi | |
| Coverage_Fake: | |
| runs-on: ubuntu-latest | |
| needs: [Nemo_CICD_Test, pre-flight] | |
| if: | | |
| ( | |
| needs.pre-flight.outputs.docs_only == 'true' | |
| || needs.pre-flight.outputs.is_deployment_workflow == 'true' | |
| ) | |
| && needs.pre-flight.outputs.is_ci_workload == 'false' | |
| && !cancelled() | |
| environment: nemo-ci | |
| steps: | |
| - name: Generate fake coverage report | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{ secrets.PAT }} | |
| script: | | |
| await github.rest.repos.createCommitStatus({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| sha: context.sha, | |
| state: 'success', | |
| description: 'No code changes - coverage check skipped', | |
| context: 'codecov/patch' | |
| }); | |
| Coverage: | |
| runs-on: ubuntu-latest | |
| needs: [pre-flight, Nemo_CICD_Test] | |
| if: | | |
| ( | |
| (needs.pre-flight.outputs.is_ci_workload == 'true' && !failure()) | |
| || success() | |
| ) | |
| && !cancelled() | |
| strategy: | |
| matrix: | |
| flag: [unit-test, e2e] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Download coverage reports of current branch | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: coverage-${{ matrix.flag }}-* | |
| - name: Get total coverage of current branch | |
| shell: bash -x -e -u -o pipefail {0} | |
| if: always() | |
| run: | | |
| pip install coverage[toml] | |
| ls -al . | |
| ls -al coverage-*/ | |
| coverage combine --keep $(ls coverage-*/.coverage) | |
| coverage report -i | |
| rm -rf coverage-* | |
| ls -al | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| verbose: true | |
| flags: ${{ matrix.flag }} | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-${{ matrix.flag }}-aggregated | |
| path: | | |
| .coverage | |
| include-hidden-files: true |