diff --git a/.github/actions/test-template/action.yml b/.github/actions/test-template/action.yml index 546f72c965..16c7280b17 100644 --- a/.github/actions/test-template/action.yml +++ b/.github/actions/test-template/action.yml @@ -11,73 +11,241 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -name: ~Build container template -on: - workflow_call: - inputs: - image-name: - required: true - type: string - description: "The name of the image to build" - dockerfile: - required: true - type: string - runner: - required: false - default: linux-amd64-gpu-rtxa6000-latest-2-nemo - type: string - description: "The runner to use for the build" - secrets: - AZURE_CLIENT_ID: - required: true - AZURE_TENANT_ID: - required: true - AZURE_SUBSCRIPTION_ID: - required: true - -jobs: - pre-flight: - runs-on: ubuntu-latest - outputs: - cache-from: ${{ steps.cache-from.outputs.LAST_PRS }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Get last merged PR - id: cache-from - env: - GH_TOKEN: ${{ github.token }} - run: | - LAST_PRS=$(gh api graphql -f query=' - query { - repository(owner: "NVIDIA", name: "NeMo-LM") { - pullRequests(states: MERGED, first: 100, orderBy: {field: UPDATED_AT, direction: DESC}) { - nodes { - number - } - } - } - }' | jq -r '.data.repository.pullRequests.nodes[].number' | while read -r number; do - echo "nemoci.azurecr.io/${{ inputs.image-name }}-buildcache:$number" - done) - - echo "LAST_PRS<> $GITHUB_OUTPUT - echo "$LAST_PRS" >> $GITHUB_OUTPUT - echo "EOF" >> $GITHUB_OUTPUT - - build: - uses: NVIDIA/NeMo-FW-CI-templates/.github/workflows/_build_container.yml@v0.29.0 - needs: [pre-flight] - with: - image-name: ${{ inputs.image-name }} - dockerfile: ${{ inputs.dockerfile }} - image-label: nemo-core - prune-filter-timerange: 24h - use-inline-cache: false - runner: ${{ inputs.runner }} - has-azure-credentials: true - secrets: - AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} +name: "Test Template" +description: "Template for running NeMo tests in a containerized environment" + +inputs: + runner: + description: "Runner to use for test" + required: true + timeout: + description: "Max runtime of test in minutes" + required: false + default: "10" + script: + description: "Test script to execute" + required: true + is_optional: + description: "Failure will cancel all other tests if set to true" + required: false + default: "false" + is_unit_test: + description: "Upload coverage as unit test" + required: false + default: "false" + image: + description: "Image to use for test" + required: false + default: "nemo_lm" + cpu-only: + description: "Run tests on CPU only" + required: false + default: "false" + azure-client-id: + description: "Azure Client ID" + required: true + azure-tenant-id: + description: "Azure Tenant ID" + required: true + azure-subscription-id: + description: "Azure Subscription ID" + required: true + has-azure-credentials: + description: "Has Azure credentials" + required: false + default: "false" + +runs: + using: "composite" + steps: + - name: Install Azure CLI + if: ${{ inputs.has-azure-credentials == 'true' }} + shell: bash + run: | + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + + - name: Azure Login + if: ${{ inputs.has-azure-credentials == 'true' }} + uses: azure/login@v2 + with: + client-id: ${{ inputs.azure-client-id }} + tenant-id: ${{ inputs.azure-tenant-id }} + subscription-id: ${{ inputs.azure-subscription-id }} + + - name: Azure ACR Login + if: ${{ inputs.has-azure-credentials == 'true' }} + shell: bash + run: | + az acr login --name nemoci + + - name: Azure Fileshare + if: ${{ inputs.has-azure-credentials == 'true' && inputs.is_unit_test == 'false' }} + shell: bash + id: azure-fileshare + run: | + sudo apt update + sudo apt install -y cifs-utils + + RESOURCE_GROUP_NAME="azure-gpu-vm-runner_group" + STORAGE_ACCOUNT_NAME="nemocistorageaccount2" + FILE_SHARE_NAME="fileshare" + + MNT_ROOT="/media" + MNT_PATH="$MNT_ROOT/$STORAGE_ACCOUNT_NAME/$FILE_SHARE_NAME" + + echo "MNT_PATH=$MNT_PATH" | tee -a "$GITHUB_OUTPUT" + + sudo mkdir -p $MNT_PATH + + # Create a folder to store the credentials for this storage account and + # any other that you might set up. + CREDENTIAL_ROOT="/etc/smbcredentials" + sudo mkdir -p "/etc/smbcredentials" + + # Get the storage account key for the indicated storage account. + # You must be logged in with az login and your user identity must have + # permissions to list the storage account keys for this command to work. + STORAGE_ACCOUNT_KEY=$(az storage account keys list \ + --resource-group $RESOURCE_GROUP_NAME \ + --account-name $STORAGE_ACCOUNT_NAME \ + --query "[0].value" --output tsv | tr -d '"') + + # Create the credential file for this individual storage account + SMB_CREDENTIAL_FILE="$CREDENTIAL_ROOT/$STORAGE_ACCOUNT_NAME.cred" + if [ ! -f $SMB_CREDENTIAL_FILE ]; then + echo "username=$STORAGE_ACCOUNT_NAME" | sudo tee $SMB_CREDENTIAL_FILE > /dev/null + echo "password=$STORAGE_ACCOUNT_KEY" | sudo tee -a $SMB_CREDENTIAL_FILE > /dev/null + else + echo "The credential file $SMB_CREDENTIAL_FILE already exists, and was not modified." + fi + + # Change permissions on the credential file so only root can read or modify the password file. + sudo chmod 600 $SMB_CREDENTIAL_FILE + + # This command assumes you have logged in with az login + HTTP_ENDPOINT=$(az storage account show --resource-group $RESOURCE_GROUP_NAME --name $STORAGE_ACCOUNT_NAME --query "primaryEndpoints.file" --output tsv | tr -d '"') + SMB_PATH=$(echo $HTTP_ENDPOINT | cut -c7-${#HTTP_ENDPOINT})$FILE_SHARE_NAME + + STORAGE_ACCOUNT_KEY=$(az storage account keys list --resource-group $RESOURCE_GROUP_NAME --account-name $STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv | tr -d '"') + + sudo mount -t cifs $SMB_PATH $MNT_PATH -o credentials=$SMB_CREDENTIAL_FILE,serverino,nosharesock,actimeo=30,mfsymlinks + + ls -al $MNT_PATH/TestData + + - name: Docker pull image + shell: bash + run: | + docker pull nemoci.azurecr.io/${{ inputs.image }}:${{ github.run_id }} + + - name: Checkout repository + uses: actions/checkout@v2 + with: + path: NeMo-LM + + - name: Start container + shell: bash + run: | + MNT_PATH=${{ steps.azure-fileshare.outputs.mnt_path }} + + ARG=("") + if [[ "${{ inputs.cpu-only }}" == "false" ]]; then + ARG=("--runtime=nvidia --gpus all") + fi + + cmd=$(cat <&1 | tee err.log + + RUN_TEST_EOF + ) + + echo "timeout_in_seconds=$(( ${{ inputs.timeout }} * 60 ))" | tee -a "$GITHUB_OUTPUT" + echo "$cmd" | tee "job.sh" + + - name: Run main script + uses: nick-fields/retry@v3 + with: + timeout_seconds: ${{ steps.create.outputs.timeout_in_seconds }} + max_attempts: 3 + shell: bash + retry_on: timeout + command: /bin/bash job.sh + on_retry_command: /bin/bash retry_job.sh + + - name: Check result + id: check + shell: bash + run: | + docker exec nemo_container_${{ github.run_id }} coverage combine || true + docker exec nemo_container_${{ github.run_id }} coverage xml + docker cp nemo_container_${{ github.run_id }}:/workspace/.coverage .coverage + docker cp nemo_container_${{ github.run_id }}:/workspace/coverage.xml coverage.xml + + coverage_report=coverage-${{ steps.create.outputs.coverage-prefix }}-${{ github.run_id }}-$(uuidgen) + echo "coverage_report=$coverage_report" >> "$GITHUB_OUTPUT" + + IS_SUCCESS=$(tail -n 1 err.log | grep -q "Finished successfully." && echo "true" || echo "false") + + if [[ "$IS_SUCCESS" == "false" && "${{ inputs.is_optional }}" == "true" ]]; then + echo "::warning:: Test failed, but displayed as successful because it is marked as optional." + IS_SUCCESS=true + fi + + if [[ "$IS_SUCCESS" == "false" ]]; then + echo Test did not finish successfully. + exit 1 + fi + + exit $EXIT_CODE + + - name: Test coverage + shell: bash -x -e -u -o pipefail {0} + run: | + docker exec -t nemo_container_${{ github.run_id }} coverage report -i + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + if: ${{ steps.check.outputs.coverage_report != 'none' }} + with: + name: ${{ steps.check.outputs.coverage_report }} + path: | + coverage.xml + .coverage + include-hidden-files: true diff --git a/.github/workflows/cicd-main.yml b/.github/workflows/cicd-main.yml index ee61e123de..09754ed323 100644 --- a/.github/workflows/cicd-main.yml +++ b/.github/workflows/cicd-main.yml @@ -37,7 +37,6 @@ jobs: env: TESTS_TO_RUN: ${{ inputs.test_to_run }} EVENT_NAME: ${{ github.event_name }} - HAS_LABEL: ${{ github.event.label.name == 'Run CICD' }} steps: - name: Check if this is a CI workload shell: bash @@ -81,61 +80,59 @@ jobs: run: | echo "Running CI tests" - # cicd-container-build: - # uses: ./.github/workflows/_build_container.yml - # needs: [pre-flight, cicd-wait-in-queue] - # if: | - # needs.pre-flight.outputs.test_to_run != '[]' - # && needs.pre-flight.outputs.components_to_run != '[]' - # && ( - # success() - # || ( - # needs.cicd-wait-in-queue.result == 'skipped' - # && needs.pre-flight.outputs.is_ci_workload == 'true' - # ) - # ) - # && !cancelled() - # with: - # image-name: nemo_container - # dockerfile: docker/Dockerfile.ci - # secrets: - # AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} - # AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} - # AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + cicd-container-build: + uses: ./.github/workflows/_build_container.yml + needs: [pre-flight, cicd-wait-in-queue] + if: | + needs.pre-flight.outputs.test_to_run != '[]' + && needs.pre-flight.outputs.components_to_run != '[]' + && ( + success() + || ( + needs.cicd-wait-in-queue.result == 'skipped' + && needs.pre-flight.outputs.is_ci_workload == 'true' + ) + ) + && !cancelled() + with: + image-name: nemo_lm + dockerfile: docker/Dockerfile.ci + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - # cicd-unit-tests: - # strategy: - # fail-fast: false - # matrix: - # include: - # - script: L0_Unit_Tests_GPU_Export_Deploy - # runner: linux-amd64-gpu-rtxa6000-latest-2-nemo - # timeout: 30 - # - script: L0_Unit_Tests_CPU_Export_Deploy - # runner: linux-amd64-cpu16 - # cpu-only: true - # needs: [cicd-container-build] - # runs-on: ${{ matrix.runner }} - # name: ${{ matrix.script }} - # environment: nemo-ci - # steps: - # - name: Checkout - # uses: actions/checkout@v4 - # with: - # path: ${{ github.run_id }} - # - name: main - # uses: NVIDIA/NeMo-Export-Deploy/.github/actions/test-template@main - # with: - # runner: ${{ runner.name }} - # script: ${{ matrix.script }} - # timeout: ${{ matrix.timeout || 10 }} - # is_unit_test: "true" - # image: nemo_container - # cpu-only: ${{ matrix.cpu-only || false }} - # has-azure-credentials: "true" - # azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} - # azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} - # azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + cicd-unit-tests: + strategy: + fail-fast: false + matrix: + include: + - script: L0_Unit_Tests_GPU + runner: linux-amd64-gpu-rtxa6000-latest-2-nemo + timeout: 30 + - script: L0_Unit_Tests_CPU + runner: linux-amd64-cpu16 + cpu-only: true + needs: [cicd-container-build] + runs-on: ${{ matrix.runner }} + name: ${{ matrix.script }} + environment: nemo-ci + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: main + uses: ./.github/actions/test-template + with: + runner: ${{ runner.name }} + script: ${{ matrix.script }} + timeout: ${{ matrix.timeout || 10 }} + is_unit_test: "true" + image: nemo_lm + cpu-only: ${{ matrix.cpu-only || false }} + has-azure-credentials: "true" + azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} + azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} + azure-subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} # cicd-e2e-tests: # strategy: @@ -176,9 +173,9 @@ jobs: Nemo_CICD_Test: needs: - pre-flight - # - cicd-test-container-build - # - cicd-main-unit-tests - # - cicd-main-e2e-tests + - cicd-container-build + - cicd-unit-tests + # - cicd-e2e-tests if: always() runs-on: ubuntu-latest permissions: write-all @@ -191,15 +188,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} RUN_ID: ${{ github.run_id }} - HAS_LABEL: ${{ github.event.label.name == 'Run CICD' }} - IS_SCHEDULED: ${{ github.event_name == 'schedule' }} run: | # Get workflow run details and check job conclusions LATEST_ATTEMPT=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion != null) | .conclusion] | last') NUM_FAILED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "failure") | .name] | length') NUM_CANCELLED=$(gh run view $RUN_ID --json jobs -q '[.jobs[] | select(.conclusion == "cancelled") | .name] | length') - if [[ $NUM_FAILED -eq 0 && $NUM_CANCELLED -eq 0 && ("$HAS_LABEL" == "true" || "$IS_SCHEDULED" == "true") ]]; then + if [[ $NUM_FAILED -eq 0 && $NUM_CANCELLED -eq 0 ]]; then RESULT="success" elif [[ $NUM_CANCELLED -gt 0 ]]; then RESULT="cancelled" diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml deleted file mode 100644 index 22949e5419..0000000000 --- a/.github/workflows/code-formatting.yml +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2025, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -name: Isort and Black Formatting -# Incrementally reformat only changed files with black, all files with isort -# -# Replaces pre-commit.ci, since it reformats all the files. -# See issue https://github.com/pre-commit-ci/issues/issues/90 -# -# The action requires a custom token to trigger workflow after pushing reformatted files back to the branch. -# `secrets.GITHUB_TOKEN` can be used instead, but this will result -# in not running necessary checks after reformatting, which is undesirable. -# For details see https://github.com/orgs/community/discussions/25702 - -on: - pull_request_target: - paths: - - "**.py" - types: [opened, synchronize, reopened, labeled, unlabeled] - -defaults: - run: - shell: bash -x -e -u -o pipefail {0} - -jobs: - reformat_with_isort_and_black: - runs-on: ubuntu-latest - permissions: - # write permissions required to commit changes - contents: write - steps: - - name: Checkout branch - uses: actions/checkout@v4 - with: - # setup repository and ref for PRs, see - # https://github.com/EndBug/add-and-commit?tab=readme-ov-file#working-with-prs - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - # custom token is required to trigger actions after reformatting + pushing - token: ${{ secrets.NEMO_REFORMAT_TOKEN }} - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: step-security/changed-files@v45.0.1 - with: - files: | - **.py - - - name: Setup Python env - uses: actions/setup-python@v5 - with: - python-version: "3.10" - - - name: black - uses: psf/black@stable - if: ${{ steps.changed-files.outputs.any_changed == 'true' }} - with: - options: "--verbose" - # apply only to changed files (pass explicitly the files) - src: "${{ steps.changed-files.outputs.all_changed_files }}" - version: "~= 24.3" - - - name: isort - uses: isort/isort-action@v1 - if: ${{ steps.changed-files.outputs.any_changed == 'true' }} - with: - isort-version: "5.13.2" - # reformat all files with isort – safe since the whole repo is already reformatted - configuration: "" - - - uses: EndBug/add-and-commit@v9 - # Commit changes. Nothing is committed if no changes. - with: - message: Apply isort and black reformatting - commit: --signoff diff --git a/.github/workflows/code-linting.yml b/.github/workflows/code-linting.yml index 7d523d99aa..4c07cc875a 100644 --- a/.github/workflows/code-linting.yml +++ b/.github/workflows/code-linting.yml @@ -25,6 +25,10 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Install dependencies + run: | + pip install -r requirements/requirements_test.txt + - name: Get changed files id: changed-files uses: step-security/changed-files@v45.0.1 @@ -55,7 +59,6 @@ jobs: ADDITIONAL_PYLINT_ARGS="--exit-zero" fi - pip install pylint set +e pylint $ADDITIONAL_PYLINT_ARGS --output "pylintrc.txt" --rcfile ".pylintrc" ${CHANGED_FILES[@]} echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT" @@ -78,17 +81,62 @@ jobs: ADDITIONAL_FLAKE8_ARGS="" fi - pip install flake8 set +e flake8 $ADDITIONAL_FLAKE8_ARGS --output "flake8.txt" --config ".flake8" ${CHANGED_FILES[@]} echo "exit-code=$?" | tee -a "$GITHUB_OUTPUT" + - name: Run isort + id: isort + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }} + run: | + if [[ -z "$CHANGED_FILES" ]]; then + echo Nothing to lint. + echo "exit-code=0" | tee -a "$GITHUB_OUTPUT" + exit 0 + fi + + set +e + isort --check-only --diff ${CHANGED_FILES[@]} > isort.txt + ISORT_EXIT_CODE=$? + + if [[ $SKIP_LINTING == true ]]; then + echo "exit-code=0" | tee -a "$GITHUB_OUTPUT" + else + echo "exit-code=$ISORT_EXIT_CODE" | tee -a "$GITHUB_OUTPUT" + fi + + - name: Run black + id: black + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} + SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }} + run: | + if [[ -z "$CHANGED_FILES" ]]; then + echo Nothing to lint. + echo "exit-code=0" | tee -a "$GITHUB_OUTPUT" + exit 0 + fi + + set +e + black --check --diff ${CHANGED_FILES[@]} > black.txt + BLACK_EXIT_CODE=$? + + if [[ $SKIP_LINTING == true ]]; then + echo "exit-code=0" | tee -a "$GITHUB_OUTPUT" + else + echo "exit-code=$BLACK_EXIT_CODE" | tee -a "$GITHUB_OUTPUT" + fi + - name: Summary env: PYLINT: ${{ steps.pylint.outputs.exit-code == 0 }} FLAKE8: ${{ steps.flake8.outputs.exit-code == 0 }} + ISORT: ${{ steps.isort.outputs.exit-code == 0 }} + BLACK: ${{ steps.black.outputs.exit-code == 0 }} + SKIP_LINTING: ${{ contains(github.event.pull_request.labels.*.name, 'skip-linting') }} run: | - if [[ "$PYLINT" != "true" ]]; then echo "Pylint output:" | tee -a $GITHUB_STEP_SUMMARY @@ -105,7 +153,23 @@ jobs: echo '```' | tee -a $GITHUB_STEP_SUMMARY fi - if [[ "$PYLINT" != "true" || "$FLAKE8" != "true" ]]; then + if [[ "$ISORT" != "true" ]]; then + echo "isort output:" | tee -a $GITHUB_STEP_SUMMARY + + echo '```' | tee -a $GITHUB_STEP_SUMMARY + cat isort.txt | tee -a $GITHUB_STEP_SUMMARY + echo '```' | tee -a $GITHUB_STEP_SUMMARY + fi + + if [[ "$BLACK" != "true" ]]; then + echo "black output:" | tee -a $GITHUB_STEP_SUMMARY + + echo '```' | tee -a $GITHUB_STEP_SUMMARY + cat black.txt | tee -a $GITHUB_STEP_SUMMARY + echo '```' | tee -a $GITHUB_STEP_SUMMARY + fi + + if [[ "$PYLINT" != "true" || "$FLAKE8" != "true" || "$ISORT" != "true" || "$BLACK" != "true" ]]; then echo "The following directories got scanned:" | tee -a $GITHUB_STEP_SUMMARY echo '```' | tee -a $GITHUB_STEP_SUMMARY diff --git a/pyproject.toml b/pyproject.toml index dd9ed8e7b0..f42fc600ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,30 +90,6 @@ skip_string_normalization = true # `required_version` is necessary for consistency (other `black` versions will fail to reformat files) required_version = "24" target-version = ['py310', 'py311', 'py312'] -extend-exclude = ''' -# A regex preceded with ^/ will apply only to files and directories -# in the root of the project. -# include here only current collections, new collections should not be ignored -# exclude the collection once it is reformatted (due to changes in PRs) -( - ^\/docs\/ - | ^\/external\/ - | ^\/examples\/ - | ^\/nemo\/collections\/asr\/ - | ^\/nemo\/collections\/common\/ - | ^\/nemo\/collections\/multimodal\/ - | ^\/nemo\/collections\/nlp\/ - | ^\/nemo\/collections\/tts\/ - | ^\/nemo\/collections\/vision\/ - | ^\/nemo\/core\/ - | ^\/nemo\/utils\/ - | ^\/scripts\/ - | ^\/tests\/ - | ^\/tools\/ - | ^\/tutorials\/ - | ^\/setup.py -) -''' [tool.pytest.ini_options] # durations=0 will display all tests execution time, sorted in ascending order starting from from the slowest one. diff --git a/requirements/requirements_test.txt b/requirements/requirements_test.txt index 0e5fe19709..6f51f7161f 100644 --- a/requirements/requirements_test.txt +++ b/requirements/requirements_test.txt @@ -1,8 +1,10 @@ black~=24.3 click>=8.1 coverage +flake8 isort>5.1.0,<6.0.0 parameterized +pylint pytest pytest-mock pytest-runner diff --git a/setup.py b/setup.py index fb2af4abbe..61d5b05d39 100644 --- a/setup.py +++ b/setup.py @@ -105,6 +105,7 @@ def req_file(filename, folder="requirements"): class StyleCommand(distutils_cmd.Command): """Checks overall project code style""" + __ISORT_BASE = 'isort' __BLACK_BASE = 'black' description = 'Checks overall project code style.' diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional_tests/__init__.py b/tests/functional_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/L0_Unit_Tests_CPU.sh b/tests/unit_tests/L0_Unit_Tests_CPU.sh new file mode 100644 index 0000000000..a64cf6893e --- /dev/null +++ b/tests/unit_tests/L0_Unit_Tests_CPU.sh @@ -0,0 +1,14 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +CUDA_VISIBLE_DEVICES="" coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/unit_tests -m "not pleasefixme" --cpu --with_downloads diff --git a/tests/unit_tests/L0_Unit_Tests_GPU.sh b/tests/unit_tests/L0_Unit_Tests_GPU.sh new file mode 100644 index 0000000000..de825eefa7 --- /dev/null +++ b/tests/unit_tests/L0_Unit_Tests_GPU.sh @@ -0,0 +1,14 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +CUDA_VISIBLE_DEVICES="0,1" coverage run -a --data-file=/workspace/.coverage --source=/workspace/ -m pytest tests/unit_tests -m "not pleasefixme" --with_downloads diff --git a/tests/unit_tests/__init__.py b/tests/unit_tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/conftest.py b/tests/unit_tests/conftest.py new file mode 100644 index 0000000000..d949dc3113 --- /dev/null +++ b/tests/unit_tests/conftest.py @@ -0,0 +1,108 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from pathlib import Path +from shutil import rmtree + +import pytest + + +def pytest_addoption(parser): + """ + Additional command-line arguments passed to pytest. + For now: + --cpu: use CPU during testing (DEFAULT: GPU) + --use_local_test_data: use local test data/skip downloading from URL/GitHub (DEFAULT: False) + """ + parser.addoption( + '--cpu', action='store_true', help="pass that argument to use CPU during testing (DEFAULT: False = GPU)" + ) + parser.addoption( + '--with_downloads', + action='store_true', + help="pass this argument to active tests which download models from the cloud.", + ) + + +@pytest.fixture +def device(request): + """Simple fixture returning string denoting the device [CPU | GPU]""" + if request.config.getoption("--cpu"): + return "CPU" + else: + return "GPU" + + +@pytest.fixture(autouse=True) +def run_only_on_device_fixture(request, device): + """Fixture to skip tests based on the device""" + if request.node.get_closest_marker('run_only_on'): + if request.node.get_closest_marker('run_only_on').args[0] != device: + pytest.skip('skipped on this device: {}'.format(device)) + + +@pytest.fixture(autouse=True) +def downloads_weights(request, device): + """Fixture to validate if the with_downloads flag is passed if necessary""" + if request.node.get_closest_marker('with_downloads'): + if not request.config.getoption("--with_downloads"): + pytest.skip( + 'To run this test, pass --with_downloads option. It will download (and cache) models from cloud.' + ) + + +@pytest.fixture(autouse=True) +def cleanup_local_folder(): + """Cleanup local experiments folder""" + # Asserts in fixture are not recommended, but I'd rather stop users from deleting expensive training runs + assert not Path("./NeMo_experiments").exists() + assert not Path("./nemo_experiments").exists() + + yield + + if Path("./NeMo_experiments").exists(): + rmtree('./NeMo_experiments', ignore_errors=True) + if Path("./nemo_experiments").exists(): + rmtree('./nemo_experiments', ignore_errors=True) + + +@pytest.fixture(autouse=True) +def reset_env_vars(): + """Reset environment variables""" + # Store the original environment variables before the test + original_env = dict(os.environ) + + # Run the test + yield + + # After the test, restore the original environment + os.environ.clear() + os.environ.update(original_env) + + +def pytest_configure(config): + """ + Initial configuration of conftest. + The function checks if test_data.tar.gz is present in tests/.data. + If so, compares its size with github's test_data.tar.gz. + If file absent or sizes not equal, function downloads the archive from github and unpacks it. + """ + config.addinivalue_line( + "markers", + "run_only_on(device): runs the test only on a given device [CPU | GPU]", + ) + config.addinivalue_line( + "markers", + "with_downloads: runs the test using data present in tests/.data", + ) diff --git a/tests/unit_tests/test_placeholder.py b/tests/unit_tests/test_placeholder.py new file mode 100644 index 0000000000..19fdab155c --- /dev/null +++ b/tests/unit_tests/test_placeholder.py @@ -0,0 +1,18 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_placeholder(): + """Should be True""" + assert True