replace component image vars with Helm values file

ArangoGutierrez · ArangoGutierrez · commit 203941a80dd3 · 2025-12-03T12:00:14.000+01:00
Replace individual component image environment variables with a values
override file approach. Add helper scripts for generating values files
from env vars and component images. Ensure clean separation between
Helm values file (-f) and --set flags to avoid conflicts.

This reduces workflow boilerplate by ~58% and makes adding new operands
trivial (no workflow changes needed).

Signed-off-by: Carlos Eduardo Arango Gutierrez &lt;eduardoa@nvidia.com&gt;
diff --git a/.github/scripts/generate-values-overrides.sh b/.github/scripts/generate-values-overrides.sh
@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+# Usage: generate-values-overrides.sh OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE
+#
+# Generates a Helm values override file for GPU Operator component images.
+# This file can be used with `helm install -f values-overrides.yaml` to
+# override default component image versions.
+
+if [[ $# -ne 4 ]]; then
+    echo "Usage: $0 OUTPUT_FILE TOOLKIT_IMAGE DEVICE_PLUGIN_IMAGE MIG_MANAGER_IMAGE" >&2
+    echo "" >&2
+    echo "Example:" >&2
+    echo "  $0 values.yaml \\" >&2
+    echo "    ghcr.io/nvidia/container-toolkit:v1.18.0-ubuntu20.04 \\" >&2
+    echo "    ghcr.io/nvidia/k8s-device-plugin:v0.17.0-ubi8 \\" >&2
+    echo "    ghcr.io/nvidia/k8s-mig-manager:v0.10.0-ubuntu20.04" >&2
+    exit 1
+fi
+
+OUTPUT_FILE="$1"
+TOOLKIT_IMAGE="$2"
+DEVICE_PLUGIN_IMAGE="$3"
+MIG_MANAGER_IMAGE="$4"
+
+# Generate values override file
+cat > "${OUTPUT_FILE}" <<EOF
+# Generated by generate-values-overrides.sh
+# Date: $(date -u +"%Y-%m-%d %H:%M:%S UTC")
+#
+# This file overrides default GPU Operator component images with
+# specific versions for forward compatibility testing.
+
+toolkit:
+  repository: ""
+  version: ""
+  image: "${TOOLKIT_IMAGE}"
+
+devicePlugin:
+  repository: ""
+  version: ""
+  image: "${DEVICE_PLUGIN_IMAGE}"
+
+migManager:
+  repository: ""
+  version: ""
+  image: "${MIG_MANAGER_IMAGE}"
+EOF
+
+echo "Generated values override file: ${OUTPUT_FILE}"
+echo ""
+echo "=== Component Images ==="
+echo "Container Toolkit:  ${TOOLKIT_IMAGE}"
+echo "Device Plugin:      ${DEVICE_PLUGIN_IMAGE}"
+echo "MIG Manager:        ${MIG_MANAGER_IMAGE}"
+echo ""
+echo "=== File Contents ==="
+cat "${OUTPUT_FILE}"
+
diff --git a/.github/workflows/e2e-tests.yaml b/.github/workflows/e2e-tests.yaml
@@ -28,18 +28,11 @@ on:
       operator_version:
         required: true
         type: string
-      toolkit_image:
+      use_values_override:
         required: false
-        type: string
-        description: 'Full container-toolkit image path (e.g., ghcr.io/nvidia/container-toolkit:v1.18.0)'
-      device_plugin_image:
-        required: false
-        type: string
-        description: 'Full device-plugin image path'
-      mig_manager_image:
-        required: false
-        type: string
-        description: 'Full mig-manager image path'
+        type: boolean
+        default: false
+        description: 'Use values-overrides artifact for component image configuration'
     secrets:
       AWS_ACCESS_KEY_ID:
         required: true
@@ -61,28 +54,13 @@ on:
         description: 'Operator version to test (override)'
         required: false
         type: string
-      toolkit_image:
-        description: 'Override container-toolkit image'
-        required: false
-        type: string
-      device_plugin_image:
-        description: 'Override device-plugin image'
-        required: false
-        type: string
-      mig_manager_image:
-        description: 'Override mig-manager image'
-        required: false
-        type: string
 
 jobs:
   variables:
     uses: ./.github/workflows/variables.yaml
     with:
       operator_image: ${{ inputs.operator_image }}
       operator_version: ${{ inputs.operator_version }}
-      toolkit_image: ${{ inputs.toolkit_image }}
-      device_plugin_image: ${{ inputs.device_plugin_image }}
-      mig_manager_image: ${{ inputs.mig_manager_image }}
 
   e2e-tests-containerd:
     needs: [variables]
@@ -93,6 +71,12 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         name: Check out code
+      - name: Download values override file
+        if: ${{ inputs.use_values_override }}
+        uses: actions/download-artifact@v5
+        with:
+          name: values-overrides
+          path: ${{ github.workspace }}
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@v0.2.17
         with:
@@ -109,13 +93,13 @@ jobs:
         run: |
           echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
           echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
+          if [[ "${{ inputs.use_values_override }}" == "true" ]]; then
+            echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> $GITHUB_ENV
+          fi
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
-          TOOLKIT_CONTAINER_IMAGE: ${{ needs.variables.outputs.toolkit_image }}
-          DEVICE_PLUGIN_IMAGE: ${{ needs.variables.outputs.device_plugin_image }}
-          MIG_MANAGER_IMAGE: ${{ needs.variables.outputs.mig_manager_image }}
           GPU_PRODUCT_NAME: "Tesla-T4"
           SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
@@ -142,6 +126,12 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         name: Check out code
+      - name: Download values override file
+        if: ${{ inputs.use_values_override }}
+        uses: actions/download-artifact@v5
+        with:
+          name: values-overrides
+          path: ${{ github.workspace }}
       - name: Set up Holodeck
         uses: NVIDIA/holodeck@v0.2.17
         with:
@@ -158,13 +148,13 @@ jobs:
         run: |
           echo "instance_hostname=ubuntu@${{ steps.get_public_dns_name.outputs.result }}" >> $GITHUB_ENV
           echo "private_key=${{ github.workspace }}/key.pem" >> $GITHUB_ENV
+          if [[ "${{ inputs.use_values_override }}" == "true" ]]; then
+            echo "VALUES_FILE=${{ github.workspace }}/values-overrides.yaml" >> $GITHUB_ENV
+          fi
       - name: Run e2e tests
         env:
           OPERATOR_VERSION: ${{ needs.variables.outputs.operator_version }}
           OPERATOR_IMAGE: ${{ needs.variables.outputs.operator_image }}
-          TOOLKIT_CONTAINER_IMAGE: ${{ needs.variables.outputs.toolkit_image }}
-          DEVICE_PLUGIN_IMAGE: ${{ needs.variables.outputs.device_plugin_image }}
-          MIG_MANAGER_IMAGE: ${{ needs.variables.outputs.mig_manager_image }}
           GPU_PRODUCT_NAME: "Tesla-T4"
           SKIP_LAUNCH: "true"
           CONTAINER_RUNTIME: "containerd"
diff --git a/.github/workflows/forward-compatibility.yaml b/.github/workflows/forward-compatibility.yaml
@@ -26,10 +26,6 @@ concurrency:
 jobs:
   fetch-latest-images:
     runs-on: ubuntu-latest
-    outputs:
-      toolkit_image: ${{ steps.images.outputs.toolkit_image }}
-      device_plugin_image: ${{ steps.images.outputs.device_plugin_image }}
-      mig_manager_image: ${{ steps.images.outputs.mig_manager_image }}
     steps:
       - uses: actions/checkout@v5
       
@@ -41,38 +37,41 @@ jobs:
           chmod +x bin/regctl
           echo "$(pwd)/bin" >> $GITHUB_PATH
       
-      - name: Get latest component images
-        id: images
+      - name: Get latest component images and generate values override file
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Fetch latest images from component repositories
           echo "::notice::Fetching latest container-toolkit image..."
           TOOLKIT=$(.github/scripts/get-latest-images.sh toolkit)
-          echo "toolkit_image=${TOOLKIT}" >> $GITHUB_OUTPUT
           
           echo "::notice::Fetching latest device-plugin image..."
           DEVICE_PLUGIN=$(.github/scripts/get-latest-images.sh device-plugin)
-          echo "device_plugin_image=${DEVICE_PLUGIN}" >> $GITHUB_OUTPUT
           
           echo "::notice::Fetching latest mig-manager image..."
           MIG_MANAGER=$(.github/scripts/get-latest-images.sh mig-manager)
-          echo "mig_manager_image=${MIG_MANAGER}" >> $GITHUB_OUTPUT
           
-          echo "::notice::=== Forward Compatibility Test Configuration ==="
-          echo "::notice::Container Toolkit: ${TOOLKIT}"
-          echo "::notice::Device Plugin: ${DEVICE_PLUGIN}"
-          echo "::notice::MIG Manager: ${MIG_MANAGER}"
+          # Generate values override file
+          .github/scripts/generate-values-overrides.sh \
+            values-overrides.yaml \
+            "${TOOLKIT}" \
+            "${DEVICE_PLUGIN}" \
+            "${MIG_MANAGER}"
+      
+      - name: Upload values override file
+        uses: actions/upload-artifact@v5
+        with:
+          name: values-overrides
+          path: values-overrides.yaml
+          retention-days: 30
 
   run-e2e-tests:
     needs: [fetch-latest-images]
     uses: ./.github/workflows/e2e-tests.yaml
     with:
       operator_image: ghcr.io/nvidia/gpu-operator
       operator_version: main-latest
-      toolkit_image: ${{ needs.fetch-latest-images.outputs.toolkit_image }}
-      device_plugin_image: ${{ needs.fetch-latest-images.outputs.device_plugin_image }}
-      mig_manager_image: ${{ needs.fetch-latest-images.outputs.mig_manager_image }}
+      use_values_override: true
     secrets: inherit
 
   notify-failure:
@@ -95,9 +94,7 @@ jobs:
               *Trigger:* ${{ github.event_name }}
               
               *Tested Components:*
-              • Container Toolkit: `${{ needs.fetch-latest-images.outputs.toolkit_image }}`
-              • Device Plugin: `${{ needs.fetch-latest-images.outputs.device_plugin_image }}`
-              • MIG Manager: `${{ needs.fetch-latest-images.outputs.mig_manager_image }}`
+              Download `values-overrides` artifact to see tested component versions
               
               *Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View Failed Run>
               <@S095E7BNGJU> 
diff --git a/.github/workflows/variables.yaml b/.github/workflows/variables.yaml
@@ -23,18 +23,6 @@ on:
         description: 'Operator version to use (optional override)'
         required: false
         type: string
-      toolkit_image:
-        description: 'Full container-toolkit image path (optional)'
-        required: false
-        type: string
-      device_plugin_image:
-        description: 'Full device-plugin image path (optional)'
-        required: false
-        type: string
-      mig_manager_image:
-        description: 'Full mig-manager image path (optional)'
-        required: false
-        type: string
     outputs:
       commit_short_sha:
         description: "The short SHA to use as a version string"
@@ -57,15 +45,6 @@ on:
       operator_image:
         description: "The operator image (with override support)"
         value: ${{ jobs.variables.outputs.operator_image }}
-      toolkit_image:
-        description: "The container-toolkit image override"
-        value: ${{ jobs.variables.outputs.toolkit_image }}
-      device_plugin_image:
-        description: "The device-plugin image override"
-        value: ${{ jobs.variables.outputs.device_plugin_image }}
-      mig_manager_image:
-        description: "The mig-manager image override"
-        value: ${{ jobs.variables.outputs.mig_manager_image }}
 
 jobs:
   variables:
@@ -78,9 +57,6 @@ jobs:
       operator_image_base: ${{ steps.vars.outputs.operator_image_base }}
       operator_version: ${{ steps.vars.outputs.operator_version }}
       operator_image: ${{ steps.vars.outputs.operator_image }}
-      toolkit_image: ${{ steps.vars.outputs.toolkit_image }}
-      device_plugin_image: ${{ steps.vars.outputs.device_plugin_image }}
-      mig_manager_image: ${{ steps.vars.outputs.mig_manager_image }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v5
@@ -124,11 +100,6 @@ jobs:
             OPERATOR_IMAGE="${OPERATOR_IMAGE_BASE}"
           fi
           
-          # Component images (optional overrides)
-          TOOLKIT_IMAGE="${{ inputs.toolkit_image }}"
-          DEVICE_PLUGIN_IMAGE="${{ inputs.device_plugin_image }}"
-          MIG_MANAGER_IMAGE="${{ inputs.mig_manager_image }}"
-          
           # Output all variables
           echo "commit_short_sha=${COMMIT_SHORT_SHA}" >> $GITHUB_OUTPUT
           echo "repo_full_name=${REPO_FULL_NAME}" >> $GITHUB_OUTPUT
@@ -137,21 +108,9 @@ jobs:
           echo "operator_image_base=${OPERATOR_IMAGE_BASE}" >> $GITHUB_OUTPUT
           echo "operator_version=${OPERATOR_VERSION}" >> $GITHUB_OUTPUT
           echo "operator_image=${OPERATOR_IMAGE}" >> $GITHUB_OUTPUT
-          echo "toolkit_image=${TOOLKIT_IMAGE}" >> $GITHUB_OUTPUT
-          echo "device_plugin_image=${DEVICE_PLUGIN_IMAGE}" >> $GITHUB_OUTPUT
-          echo "mig_manager_image=${MIG_MANAGER_IMAGE}" >> $GITHUB_OUTPUT
           
           # Display for debugging
           echo "::notice::Commit SHA: ${COMMIT_SHORT_SHA}"
           echo "::notice::Push on build: ${PUSH_ON_BUILD}"
           echo "::notice::Operator image: ${OPERATOR_IMAGE}:${OPERATOR_VERSION}"
-          if [[ -n "${TOOLKIT_IMAGE}" ]]; then
-            echo "::notice::Using custom toolkit: ${TOOLKIT_IMAGE}"
-          fi
-          if [[ -n "${DEVICE_PLUGIN_IMAGE}" ]]; then
-            echo "::notice::Using custom device-plugin: ${DEVICE_PLUGIN_IMAGE}"
-          fi
-          if [[ -n "${MIG_MANAGER_IMAGE}" ]]; then
-            echo "::notice::Using custom mig-manager: ${MIG_MANAGER_IMAGE}"
-          fi
 
diff --git a/tests/scripts/env-to-values.sh b/tests/scripts/env-to-values.sh
diff --git a/tests/scripts/install-operator.sh b/tests/scripts/install-operator.sh
