rename #21373
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. | |
| # SPDX-License-Identifier: Apache-2.0 | |
| name: NVIDIA Dynamo Github Validation | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release/*.*.* | |
| - "pull-request/[0-9]+" | |
| concurrency: | |
| # The group name is a ternary operation. If the ref_name is 'main', | |
| # then the group name uses the run_id to ensure a unique group for | |
| # 'main' pushes. Otherwise, the group name is the ref_name, so that | |
| # workflows on the same PR/branch have the same group name for cancelling. | |
| group: dynamo-build-test-${{ github.ref_name == 'main' && github.run_id || github.ref_name }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} | |
| env: | |
| CUDA_VERSION: '12.9' | |
| BUILDER_NAME: b-${{ github.run_id }}-${{ github.run_attempt }} | |
| jobs: | |
| changed-files: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| core: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.core == 'true' }} | |
| planner: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.planner == 'true' }} | |
| frontend: ${{ github.ref_name == 'main' || startsWith(github.ref_name, 'release/') || steps.changes.outputs.frontend == 'true' }} | |
| builder_name: ${{ steps.export-builder-name.outputs.builder_name }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check for changes | |
| id: changes | |
| uses: ./.github/actions/changed-files | |
| with: | |
| gh_token: ${{ github.token }} | |
| - name: Export builder name | |
| id: export-builder-name | |
| run: | | |
| echo "builder_name=${{ env.BUILDER_NAME }}" >> $GITHUB_OUTPUT | |
| dynamo-status-check: | |
| runs-on: ubuntu-latest | |
| needs: [changed-files, build, rust-checks, test-parallel, test-sequential] | |
| if: always() | |
| steps: | |
| - name: "Check all dependent jobs" | |
| run: | | |
| echo '${{ toJson(needs) }}' | jq -e 'to_entries | map(.value.result) | all(. as $result | ["success", "skipped"] | any($result == .))' | |
| build: | |
| needs: changed-files | |
| if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true' | |
| runs-on: prod-builder-v3 | |
| name: Build | |
| timeout-minutes: 60 | |
| outputs: | |
| # Only pass the non-secret tag suffix between jobs (GitHub blanks outputs containing secrets) | |
| runtime_tag_suffix: ${{ steps.define_image_tag.outputs.runtime_tag_suffix }} | |
| test_tag_suffix: ${{ steps.define_image_tag.outputs.test_tag_suffix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| with: | |
| lfs: true | |
| - name: Initialize Dynamo Builder | |
| uses: ./.github/actions/init-dynamo-builder | |
| with: | |
| builder_name: ${{ needs.changed-files.outputs.builder_name }} | |
| flavor: general | |
| arch: amd64 | |
| - name: Docker Login | |
| if: github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'push' | |
| uses: ./.github/actions/docker-login | |
| with: | |
| ngc_ci_access_token: ${{ secrets.NGC_CI_ACCESS_TOKEN }} | |
| aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} | |
| azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }} | |
| azure_acr_user: ${{ secrets.AZURE_ACR_USER }} | |
| azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }} | |
| - name: Define Image Tag | |
| id: define_image_tag | |
| env: | |
| ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com | |
| run: | | |
| CUDA_MAJOR=${CUDA_VERSION%%.*} | |
| RUNTIME_TAG_SUFFIX="${{ github.sha }}-dynamo-runtime-cuda${CUDA_MAJOR}-amd64" | |
| TEST_TAG_SUFFIX="${{ github.sha }}-dynamo-test-cuda${CUDA_MAJOR}-amd64" | |
| echo "runtime_tag_suffix=${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT | |
| echo "test_tag_suffix=${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT | |
| echo "runtime_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${RUNTIME_TAG_SUFFIX}" >> $GITHUB_OUTPUT | |
| echo "test_remote_tag=${ECR_HOSTNAME}/ai-dynamo/dynamo:${TEST_TAG_SUFFIX}" >> $GITHUB_OUTPUT | |
| - name: Generate Dockerfile | |
| shell: bash | |
| run: | | |
| echo "Generating Dockerfile for target: runtime and framework: dynamo" | |
| python ./container/render.py \ | |
| --target=runtime \ | |
| --framework=dynamo \ | |
| --platform=amd64 \ | |
| --cuda-version=${{ env.CUDA_VERSION }} \ | |
| --show-result \ | |
| --output-short-filename | |
| - name: Build and Push Runtime Image | |
| uses: ./.github/actions/docker-remote-build | |
| with: | |
| image_tag: ${{ steps.define_image_tag.outputs.runtime_remote_tag }} | |
| framework: dynamo | |
| target: runtime | |
| platform: amd64 | |
| cuda_version: ${{ env.CUDA_VERSION }} | |
| ci_token: ${{ secrets.CI_TOKEN }} | |
| aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} | |
| sccache_s3_bucket: ${{ secrets.SCCACHE_S3_BUCKET }} | |
| aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| push_image: 'true' | |
| - name: Build and Push Test Image | |
| env: | |
| ECR_HOSTNAME: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com | |
| shell: bash | |
| run: | | |
| CACHE_TAG="test-dynamo-cuda${CUDA_VERSION%%.*}-amd64-cache" | |
| CACHE_ARGS="--cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG}" | |
| CACHE_ARGS+=" --cache-from type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG}" | |
| if [[ "$GITHUB_REF_NAME" == "main" ]]; then | |
| CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:main-${CACHE_TAG},mode=max" | |
| elif [[ "$GITHUB_REF_NAME" =~ ^release ]]; then | |
| CACHE_ARGS+=" --cache-to type=registry,ref=${ECR_HOSTNAME}/ai-dynamo/dynamo:release-${CACHE_TAG},mode=max" | |
| fi | |
| docker buildx build \ | |
| --progress=plain \ | |
| --push \ | |
| --platform linux/amd64 \ | |
| -f container/Dockerfile.test \ | |
| --build-arg BASE_IMAGE=${{ steps.define_image_tag.outputs.runtime_remote_tag }} \ | |
| ${CACHE_ARGS} \ | |
| -t ${{ steps.define_image_tag.outputs.test_remote_tag }} . | |
| rust-checks: | |
| needs: [changed-files, build] | |
| if: needs.changed-files.outputs.core == 'true' | |
| runs-on: prod-tester-amd-gpu-v1 | |
| name: Rust Checks | |
| timeout-minutes: 30 | |
| env: | |
| CONTAINER_ID: test_${{ github.run_id }}_${{ github.run_attempt }}_rust_dynamo | |
| IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.runtime_tag_suffix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Docker Login | |
| uses: ./.github/actions/docker-login | |
| with: | |
| aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} | |
| - name: Pull runtime image | |
| run: | | |
| source ./.github/scripts/retry_docker.sh | |
| retry_pull ${{ env.IMAGE_TAG }} | |
| - name: Run Rust checks (block-manager + media-ffmpeg + integration tests) | |
| run: | | |
| docker run --rm --runtime=nvidia --gpus all --user root -w /workspace/lib/llm \ | |
| --name ${{ env.CONTAINER_ID }}_rust_checks \ | |
| -e SCCACHE_BUCKET=${{ secrets.SCCACHE_S3_BUCKET }} \ | |
| -e SCCACHE_REGION=${{ secrets.AWS_DEFAULT_REGION }} \ | |
| -e AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \ | |
| -e AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \ | |
| ${{ env.IMAGE_TAG }} \ | |
| bash -ec 'ARCH_ALT=x86_64 /workspace/container/use-sccache.sh install && \ | |
| eval $(/workspace/container/use-sccache.sh setup-env) && \ | |
| rustup component add rustfmt clippy && \ | |
| cargo fmt -- --check && \ | |
| cargo clippy --features block-manager,media-ffmpeg,testing-nixl,integration --no-deps --all-targets -- -D warnings && \ | |
| cargo test --locked --all-targets --features=block-manager,media-ffmpeg,testing-nixl,integration -- --nocapture && \ | |
| cargo clippy -p kvbm-physical --no-deps --all-targets -- -D warnings && \ | |
| cargo test --locked -p kvbm-physical --features testing-kvbm -- --nocapture --test-threads=4 && \ | |
| /workspace/container/use-sccache.sh show-stats "Rust Checks"' | |
| test-parallel: | |
| needs: [changed-files, build] | |
| if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true' | |
| runs-on: prod-builder-amd-v1 | |
| name: Pytest (parallel) | |
| timeout-minutes: 30 | |
| env: | |
| IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Docker Login | |
| uses: ./.github/actions/docker-login | |
| with: | |
| aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} | |
| - name: Pull test image | |
| run: | | |
| source ./.github/scripts/retry_docker.sh | |
| retry_pull ${{ env.IMAGE_TAG }} | |
| - name: Run pytest (parallel tests with xdist) | |
| uses: ./.github/actions/pytest | |
| with: | |
| image_tag: ${{ env.IMAGE_TAG }} | |
| pytest_marks: "pre_merge and parallel and not (vllm or sglang or trtllm) and (gpu_0)" | |
| framework: dynamo | |
| test_type: "pre_merge_parallel" | |
| platform_arch: amd64 | |
| enable_mypy: 'true' | |
| hf_token: ${{ secrets.HF_TOKEN }} | |
| parallel_mode: '4' | |
| dind_as_sidecar: 'false' | |
| test-sequential: | |
| needs: [changed-files, build] | |
| if: needs.changed-files.outputs.core == 'true' || needs.changed-files.outputs.planner == 'true' || needs.changed-files.outputs.frontend == 'true' | |
| runs-on: prod-builder-amd-v1 | |
| name: Pytest (sequential) | |
| timeout-minutes: 30 | |
| env: | |
| IMAGE_TAG: ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.test_tag_suffix }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 | |
| - name: Docker Login | |
| uses: ./.github/actions/docker-login | |
| with: | |
| aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }} | |
| aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }} | |
| - name: Pull test image | |
| run: | | |
| source ./.github/scripts/retry_docker.sh | |
| retry_pull ${{ env.IMAGE_TAG }} | |
| - name: Run pytest (sequential tests) | |
| uses: ./.github/actions/pytest | |
| with: | |
| image_tag: ${{ env.IMAGE_TAG }} | |
| pytest_marks: "pre_merge and not parallel and not (vllm or sglang or trtllm) and (gpu_0)" | |
| framework: dynamo | |
| test_type: "pre_merge_sequential" | |
| platform_arch: amd64 | |
| enable_mypy: 'false' | |
| hf_token: ${{ secrets.HF_TOKEN }} | |
| parallel_mode: 'none' | |
| dind_as_sidecar: 'false' |