ci: add a preliminary compliance scan to ci (#7289) (#7306)

Signed-off-by: Harrison King Saturley-Hall <hsaturleyhal@nvidia.com>
Signed-off-by: Anant Sharma <anants@nvidia.com>
Co-authored-by: Anant Sharma <anants@nvidia.com>

Author: saturley-hall

.dockerignore

@@ -1,17 +1,5 @@
 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 **/*.onnx
 **/*.plan
@@ -45,6 +33,7 @@ container/Dockerfile*
 container/**/*.Dockerfile
 container/render.py
 container/context.yaml
+container/compliance/
 .venv
 .venv-docs
 

.github/actions/compliance-scan/action.yml

@@ -0,0 +1,88 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+name: 'Compliance Scan'
+description: 'Generate attribution CSVs (dpkg + Python) for a container image and upload as workflow artifacts'
+
+inputs:
+  image:
+    description: 'Full container image URI to scan (must be pullable)'
+    required: true
+  artifact_name:
+    description: 'Name for the uploaded artifact (e.g., compliance-vllm-cuda12-amd64)'
+    required: true
+  framework:
+    description: 'Framework name for base image resolution (vllm, sglang, trtllm, dynamo)'
+    required: false
+    default: ''
+  target:
+    description: 'Build target for base image resolution (runtime or frontend)'
+    required: false
+    default: 'runtime'
+  cuda_version:
+    description: 'CUDA version for base image resolution (e.g., 12.9, 13.0, 13.1)'
+    required: false
+    default: ''
+  base_image:
+    description: 'Explicit base image for diff (overrides framework/cuda-version auto-resolve)'
+    required: false
+    default: ''
+  retention_days:
+    description: 'Artifact retention in days'
+    required: false
+    default: '90'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 #v3.11.1
+      with:
+        driver: docker-container
+        # Enable BuildKit for enhanced metadata
+        buildkitd-flags: --debug
+        version: v0.14.1
+    - name: Cleanup
+      if: always()
+      shell: bash
+      run: |
+        docker system prune -af
+    - name: Set up Python
+      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
+      with:
+        python-version: '3.12'
+        pip-install: pyyaml
+
+    - name: Pull container image
+      shell: bash
+      run: |
+        source ./.github/scripts/retry_docker.sh
+        retry_pull ${{ inputs.image }}
+
+    - name: Generate attribution CSVs
+      shell: bash
+      run: |
+        ARGS=""
+        if [ -n "${{ inputs.base_image }}" ]; then
+          ARGS+=" --base-image ${{ inputs.base_image }}"
+        elif [ -n "${{ inputs.framework }}" ]; then
+          ARGS+=" --framework ${{ inputs.framework }}"
+          ARGS+=" --target ${{ inputs.target }}"
+          if [ -n "${{ inputs.cuda_version }}" ]; then
+            ARGS+=" --cuda-version ${{ inputs.cuda_version }}"
+          fi
+        fi
+
+        python container/compliance/generate_attributions.py \
+          "${{ inputs.image }}" \
+          --output "${{ inputs.artifact_name }}.csv" \
+          --verbose \
+          ${ARGS}
+
+    - name: Upload attribution artifacts
+      if: always()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4.6.2
+      with:
+        name: ${{ inputs.artifact_name }}
+        path: ${{ inputs.artifact_name }}*.csv
+        retention-days: ${{ inputs.retention_days }}

.github/filters.yaml

@@ -82,6 +82,7 @@ core:
   - 'container/templates/wheel_builder.Dockerfile'
   - '.dockerignore'
   - 'container/deps/*'
+  - 'container/compliance/**'
   - '.cargo/config.toml'
   - 'lib/**'
   - 'tests/**'
@@ -151,6 +152,7 @@ frontend:
   - '*.toml'
   - '*.lock'
   - 'container/deps/*'
+  - 'container/compliance/**'
   - 'components/src/dynamo/router/**'
   - 'components/src/dynamo/mocker/**'
   - 'components/src/dynamo/frontend/**'

.github/scripts/retry_docker.sh

@@ -0,0 +1,45 @@
+# Retry docker operations with exponential backoff.
+# Safe under `set -e`: the `if` conditional context prevents a failed
+# `docker <operation>` from triggering an immediate exit.
+retry_docker_operation() {
+  local operation="$1"
+  local image="$2"
+  local max_attempts=3
+  local wait_seconds=10
+  local attempt=1
+
+  if [[ "$operation" != "push" && "$operation" != "pull" ]]; then
+    echo "Unsupported docker operation: $operation (expected: push|pull)" >&2
+    return 2
+  fi
+
+  while true; do
+    if docker "$operation" "$image"; then
+      return 0
+    fi
+    echo "Docker ${operation} failed for $image (attempt ${attempt}/${max_attempts})." >&2
+
+    if (( attempt >= max_attempts )); then
+      echo "Docker ${operation} failed after ${max_attempts} attempts: $image" >&2
+      return 1
+    fi
+
+    echo "Retrying docker ${operation} in ${wait_seconds}s..."
+    sleep "$wait_seconds"
+    attempt=$((attempt + 1))
+    wait_seconds=$((wait_seconds * 2))
+    if (( wait_seconds > 120 )); then
+      wait_seconds=120
+    fi
+  done
+}
+
+retry_push() {
+  local image="$1"
+  retry_docker_operation push "$image"
+}
+
+retry_pull() {
+  local image="$1"
+  retry_docker_operation pull "$image"
+}

.github/workflows/build-frontend-image.yaml

@@ -221,6 +221,44 @@ jobs:
           echo "| \`${{ steps.calculate-target-tag.outputs.default_target_image_uri }}\` |" >> $GITHUB_STEP_SUMMARY
           echo "| \`${{ steps.calculate-target-tag.outputs.azure_target_image_uri }}\` |" >> $GITHUB_STEP_SUMMARY
 
+  # ============================================================================
+  # COMPLIANCE — Generate attribution CSVs for dpkg and Python packages
+  # ============================================================================
+  compliance:
+    needs: [build-frontend-image, changed-files]
+    if: needs.build-frontend-image.result == 'success'
+    strategy:
+      fail-fast: false
+      matrix:
+        arch: [amd64, arm64]
+    name: Compliance frontend-${{ matrix.arch }}
+    runs-on: ${{ matrix.arch == 'amd64' && 'prod-builder-amd-v1' || 'prod-tester-arm-v1' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+      - name: Docker Login
+        uses: ./.github/actions/docker-login
+        with:
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Calculate image URI
+        id: images
+        shell: bash
+        run: |
+          TARGET_TAG="${{ github.sha }}-frontend-${{ matrix.arch }}"
+          FRONTEND_IMAGE="${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${TARGET_TAG}"
+          echo "frontend_image=${FRONTEND_IMAGE}" >> $GITHUB_OUTPUT
+      - name: Compliance scan
+        uses: ./.github/actions/compliance-scan
+        with:
+          image: ${{ steps.images.outputs.frontend_image }}
+          artifact_name: compliance-frontend-${{ matrix.arch }}
+          framework: dynamo
+          target: frontend
+
   frontend-status-check:
     runs-on: ubuntu-latest
     needs: [changed-files, build-frontend-image, build-epp-image]

.github/workflows/build-test-distribute-flavor.yml

@@ -427,6 +427,43 @@ jobs:
           dind_as_sidecar: 'true'
 
 
+  # ============================================================================
+  # COMPLIANCE — Generate attribution CSVs for dpkg and Python packages
+  # ============================================================================
+  compliance:
+    if: inputs.build_image && inputs.push_image
+    needs: [build]
+    name: Compliance cuda${{ inputs.cuda_version }}-${{ inputs.platform }}
+    runs-on: ${{ inputs.platform == 'amd64' && 'prod-builder-amd-v1' || 'prod-tester-arm-v1' }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955  # v4.3.0
+      - name: Docker Login
+        uses: ./.github/actions/docker-login
+        with:
+          aws_default_region: ${{ secrets.AWS_DEFAULT_REGION }}
+          aws_account_id: ${{ secrets.AWS_ACCOUNT_ID }}
+          azure_acr_hostname: ${{ secrets.AZURE_ACR_HOSTNAME }}
+          azure_acr_user: ${{ secrets.AZURE_ACR_USER }}
+          azure_acr_password: ${{ secrets.AZURE_ACR_PASSWORD }}
+      - name: Calculate image URI
+        id: images
+        shell: bash
+        run: |
+          CUDA_VERSION_RAW=${{ inputs.cuda_version }}
+          CUDA_VERSION=${CUDA_VERSION_RAW%%.*}
+          echo "cuda_major=${CUDA_VERSION}" >> $GITHUB_OUTPUT
+          RUNTIME_IMAGE=${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/ai-dynamo/dynamo:${{ needs.build.outputs.target_tag_plain }}-cuda${CUDA_VERSION}-${{ inputs.platform }}
+          echo "runtime_image=${RUNTIME_IMAGE}" >> $GITHUB_OUTPUT
+      - name: Compliance scan
+        uses: ./.github/actions/compliance-scan
+        with:
+          image: ${{ steps.images.outputs.runtime_image }}
+          artifact_name: compliance-${{ inputs.framework }}-${{ inputs.target }}${{ inputs.make_efa && '-efa' || '' }}-cuda${{ steps.images.outputs.cuda_major }}-${{ inputs.platform }}
+          framework: ${{ inputs.framework }}
+          cuda_version: ${{ inputs.cuda_version }}
+
+
   # ============================================================================
   # COPY TO ACR
   # ============================================================================

container/compliance/README.md

@@ -0,0 +1,107 @@
+# Container Compliance Tooling
+
+Scripts for generating attribution CSVs from built container images, listing all installed dpkg and Python packages with their SPDX license identifiers where known.
+
+## Output format
+
+Each run produces up to two CSV files:
+
+| Column | Description |
+|--------|-------------|
+| `package_name` | Package name as reported by dpkg or pip |
+| `version` | Installed version |
+| `type` | `dpkg` or `python` |
+| `spdx_license` | SPDX identifier (e.g. `MIT`, `Apache-2.0`) or `UNKNOWN` |
+
+Files are sorted by `(type, package_name)` for stable diffs.
+
+When a base image is provided, a second `_diff.csv` file is written containing only packages that are new or version-changed relative to the base — i.e. what Dynamo's build layers added on top of the upstream image.
+
+## Usage
+
+```bash
+# Full scan, output to stdout
+python container/compliance/generate_attributions.py <image:tag>
+
+# Write to file
+python container/compliance/generate_attributions.py <image:tag> -o attribution.csv
+
+# With base image diff — auto-resolved from context.yaml
+python container/compliance/generate_attributions.py <image:tag> \
+    --framework vllm \
+    --cuda-version 12.9 \
+    -o attribution-vllm-cuda12-amd64.csv
+# Produces: attribution-vllm-cuda12-amd64.csv  (full)
+#           attribution-vllm-cuda12-amd64_diff.csv  (delta from base)
+
+# With explicit base image override
+python container/compliance/generate_attributions.py <image:tag> \
+    --base-image nvcr.io/nvidia/cuda:12.9.1-runtime-ubuntu24.04 \
+    -o attribution.csv
+
+# Frontend image
+python container/compliance/generate_attributions.py <image:tag> \
+    --framework dynamo \
+    --target frontend \
+    -o attribution-frontend-amd64.csv
+
+# dpkg only
+python container/compliance/generate_attributions.py <image:tag> \
+    --types dpkg \
+    -o attribution-dpkg.csv
+```
+
+### All flags
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `image` | *(required)* | Container image to scan |
+| `--output`, `-o` | stdout | Output CSV path |
+| `--framework` | — | Auto-resolve base image from `context.yaml` (`vllm`, `sglang`, `trtllm`, `dynamo`) |
+| `--target` | `runtime` | Build target for base resolution (`runtime` or `frontend`) |
+| `--cuda-version` | — | CUDA version for base resolution (e.g. `12.9`, `13.0`, `13.1`) |
+| `--base-image` | — | Explicit base image URI (overrides `--framework` auto-resolve) |
+| `--context-yaml` | `container/context.yaml` | Path to context.yaml |
+| `--types` | `dpkg,python` | Comma-separated list of types to extract |
+| `--docker-cmd` | `docker` | Docker binary to use |
+| `--verbose`, `-v` | — | Enable verbose logging to stderr |
+
+## Base image reference
+
+| Framework | CUDA | Base image |
+|-----------|------|------------|
+| `vllm` | 12.9 | `nvcr.io/nvidia/cuda:12.9.1-runtime-ubuntu24.04` |
+| `vllm` | 13.0 | `nvcr.io/nvidia/cuda:13.0.2-runtime-ubuntu24.04` |
+| `sglang` | 12.9 | `lmsysorg/sglang:v0.5.9-runtime` |
+| `sglang` | 13.0 | `lmsysorg/sglang:v0.5.9-cu130-runtime` |
+| `trtllm` | 13.1 | `nvcr.io/nvidia/cuda-dl-base:25.12-cuda13.1-runtime-ubuntu24.04` |
+| `dynamo` frontend | — | `nvcr.io/nvidia/base/ubuntu:noble-20250619` |
+
+These values are sourced from `container/context.yaml` at runtime; the table above reflects the current defaults.
+
+## How it works
+
+The script runs two lightweight helper scripts **inside the container** via `docker run --rm -v`:
+
+- **dpkg extractor** — runs `dpkg-query` to list packages, then reads `/usr/share/doc/<pkg>/copyright` files for license info. Only DEP-5 machine-readable copyright files are parsed; ambiguous cases return `UNKNOWN`.
+- **Python extractor** — uses `importlib.metadata.distributions()` to iterate installed packages. License is read from `License-Expression` (PEP 639), then `License` metadata, then trove classifiers. Ambiguous cases return `UNKNOWN`.
+
+Both helpers are self-contained and have no external dependencies — they run with whatever Python is in the container.
+
+## License detection
+
+Detection is intentionally conservative: only unambiguous matches are assigned SPDX identifiers. The `UNKNOWN` entries are expected; they can be resolved with additional analysis against the raw copyright files.
+
+## CI integration
+
+Attribution CSVs are generated automatically as part of CI after every successful image build. Artifacts are available in the GitHub Actions workflow run under:
+- `compliance-{framework}-cuda{major}-{platform}` — runtime images
+- `compliance-frontend-{arch}` — frontend image
+
+The scan runs as a separate lightweight job (`prod-default-small-v2`) in parallel with tests, so it does not extend pipeline wall time.
+
+## Requirements
+
+- Python 3.11+
+- `docker` (or compatible CLI) with access to the target registry
+- `pyyaml` — only required on the host when using `--framework`/`--cuda-version` base image auto-resolution (`pip install pyyaml`)

container/compliance/extractors/__init__.py

@@ -0,0 +1,4 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Attribution extractors for container dependency scanning."""