add talos support

Signed-off-by: hydazz <alexanderhyde@icloud.com>

Author: hydazz

.github/workflows/image.yaml

@@ -31,7 +31,7 @@ jobs:
     steps:
       - uses: actions/checkout@v5
         name: Check out code
-      
+
       - name: Install Go
         uses: actions/setup-go@v6
         with:
@@ -61,3 +61,9 @@ jobs:
         run: |
           echo "${VERSION}"
           make -f deployments/container/Makefile build
+          # Push the image again, using a shorter tag (only the 8 character
+          # commit hash). That's for consumption in downstream CI (for copying
+          # the image from GHCR to elsewhere). See
+          # https://github.com/NVIDIA/k8s-dra-driver-gpu/issues/688
+          export IMAGE_TAG="$(echo $GITHUB_SHA | cut -c1-8)"
+          make -f deployments/container/Makefile build

.nvidia-ci.yml

@@ -38,13 +38,11 @@ variables:
   # Define the public staging registry
   STAGING_REGISTRY: ghcr.io/nvidia
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
-  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
   KITMAKER_RELEASE_FOLDER: "kitmaker"
   PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 stages:
   - pull
-  - scan
   - release
   - ngc-publish
 
@@ -69,7 +67,7 @@ workflow:
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
-    - export REGCTL_VERSION=v0.4.5
+    - export REGCTL_VERSION=v0.4.8
     - apk add --no-cache curl
     - mkdir -p bin
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
@@ -146,48 +144,11 @@ pull-images:
   script:
     - echo "Skipped in internal CI"
 
-# The .scan step forms the base of the image scan operation performed before releasing
-# images.
-scan-images:
-  stage: scan
-  needs:
-  - pull-images
-  image: "${PULSE_IMAGE}"
-  parallel:
-    matrix:
-        PLATFORM: ["linux/amd64", "linux/arm64"]
-  variables:
-    IMAGE: "${CI_REGISTRY_IMAGE}/k8s-dra-driver-gpu:${CI_COMMIT_SHORT_SHA}"
-    IMAGE_ARCHIVE: "k8s-dra-driver-gpu-${CI_JOB_ID}.tar"
-  allow_failure: true
-  script:
-    - |
-      docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-      echo "Scanning image ${IMAGE} for ${PLATFORM}"
-      docker pull --platform="${PLATFORM}" "${IMAGE}"
-      docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
-      AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
-      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
-      if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
-
-      pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
-      rm -f "${IMAGE_ARCHIVE}"
-  artifacts:
-    when: always
-    expire_in: 1 week
-    paths:
-      - pulse-cli.log
-      - licenses.json
-      - sbom.json
-      - vulns.json
-      - policy_evaluation.json
 
 push-images-to-staging:
   extends:
     - .copy-images
   stage: release
-  needs:
-    - scan-images
   variables:
     IN_REGISTRY: "${CI_REGISTRY}"
     IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
@@ -204,7 +165,6 @@ push-images-to-staging:
 .publish-images:
   stage: ngc-publish
   needs:
-    - scan-images
     - push-images-to-staging
   image:
     name: "${CNT_NGC_PUBLISH_IMAGE}"
@@ -254,15 +214,15 @@ push-images-to-staging:
       - "${PROJECT_NAME}.yaml"
 
 
-publish-images-to-ngc:
+create-ngc-publish-mr:
   extends:
     - .publish-images
   rules:
     - if: $CI_COMMIT_TAG
 
 # We create a dummy MR that exercises the publishing logic.
 # TODO: This MR should be closed automatically.
-publish-images-dummy:
+create-ngc-publish-mr-dummy:
   extends:
     - .publish-images
   variables:

cmd/compute-domain-kubelet-plugin/cdi.go

@@ -46,6 +46,14 @@ const (
 	defaultCDIRoot = "/var/run/cdi"
 )
 
+func getTalosLibrarySearchPaths() []string {
+	return []string{
+		"/driver-root/usr/local/glibc/usr/lib",
+		"/driver-root/usr/local/glibc/lib",
+		"/driver-root/usr/local/glibc/lib64",
+	}
+}
+
 type CDIHandler struct {
 	logger            *logrus.Logger
 	nvml              nvml.Interface
@@ -103,6 +111,7 @@ func NewCDIHandler(opts ...cdiOption) (*CDIHandler, error) {
 			nvcdi.WithVendor(h.vendor),
 			nvcdi.WithClass(h.deviceClass),
 			nvcdi.WithNVIDIACDIHookPath(h.nvidiaCDIHookPath),
+			nvcdi.WithLibrarySearchPaths(getTalosLibrarySearchPaths()),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create CDI library for devices: %w", err)
@@ -120,6 +129,7 @@ func NewCDIHandler(opts ...cdiOption) (*CDIHandler, error) {
 			nvcdi.WithVendor(h.vendor),
 			nvcdi.WithClass(h.claimClass),
 			nvcdi.WithNVIDIACDIHookPath(h.nvidiaCDIHookPath),
+			nvcdi.WithLibrarySearchPaths(getTalosLibrarySearchPaths()),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create CDI library for claims: %w", err)

cmd/compute-domain-kubelet-plugin/root.go

@@ -34,6 +34,7 @@ func (r root) getDriverLibraryPath() (string, error) {
 		"/lib64",
 		"/lib/x86_64-linux-gnu",
 		"/lib/aarch64-linux-gnu",
+		"/usr/local/glibc/usr/lib",
 	}
 
 	libraryPath, err := r.findFile("libnvidia-ml.so.1", librarySearchPaths...)
@@ -51,6 +52,7 @@ func (r root) getNvidiaSMIPath() (string, error) {
 		"/usr/sbin",
 		"/bin",
 		"/sbin",
+		"/usr/local/bin",
 	}
 
 	binaryPath, err := r.findFile("nvidia-smi", binarySearchPaths...)

cmd/gpu-kubelet-plugin/cdi.go

@@ -46,6 +46,14 @@ const (
 	defaultCDIRoot = "/var/run/cdi"
 )
 
+func getTalosLibrarySearchPaths() []string {
+	return []string{
+		"/driver-root/usr/local/glibc/usr/lib",
+		"/driver-root/usr/local/glibc/lib",
+		"/driver-root/usr/local/glibc/lib64",
+	}
+}
+
 type CDIHandler struct {
 	logger            *logrus.Logger
 	nvml              nvml.Interface
@@ -103,6 +111,7 @@ func NewCDIHandler(opts ...cdiOption) (*CDIHandler, error) {
 			nvcdi.WithVendor(h.vendor),
 			nvcdi.WithClass(h.deviceClass),
 			nvcdi.WithNVIDIACDIHookPath(h.nvidiaCDIHookPath),
+			nvcdi.WithLibrarySearchPaths(getTalosLibrarySearchPaths()),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create CDI library for devices: %w", err)
@@ -120,6 +129,7 @@ func NewCDIHandler(opts ...cdiOption) (*CDIHandler, error) {
 			nvcdi.WithVendor(h.vendor),
 			nvcdi.WithClass(h.claimClass),
 			nvcdi.WithNVIDIACDIHookPath(h.nvidiaCDIHookPath),
+			nvcdi.WithLibrarySearchPaths(getTalosLibrarySearchPaths()),
 		)
 		if err != nil {
 			return nil, fmt.Errorf("unable to create CDI library for claims: %w", err)

cmd/gpu-kubelet-plugin/root.go

@@ -34,6 +34,7 @@ func (r root) getDriverLibraryPath() (string, error) {
 		"/lib64",
 		"/lib/x86_64-linux-gnu",
 		"/lib/aarch64-linux-gnu",
+		"/usr/local/glibc/usr/lib",
 	}
 
 	libraryPath, err := r.findFile("libnvidia-ml.so.1", librarySearchPaths...)
@@ -51,6 +52,7 @@ func (r root) getNvidiaSMIPath() (string, error) {
 		"/usr/sbin",
 		"/bin",
 		"/sbin",
+		"/usr/local/bin",
 	}
 
 	binaryPath, err := r.findFile("nvidia-smi", binarySearchPaths...)

deployments/helm/nvidia-dra-driver-gpu/templates/kubeletplugin.yaml

@@ -129,7 +129,7 @@ spec:
         - name: NVIDIA_VISIBLE_DEVICES
           value: void
         - name: CDI_ROOT
-          value: /var/run/cdi
+          value: {{ .Values.cdiRoot | quote }}
         - name: NVIDIA_MIG_CONFIG_DEVICES
           value: all
         - name: NODE_NAME
@@ -166,7 +166,7 @@ spec:
           mountPath: {{ .Values.kubeletPlugin.kubeletPluginsDirectoryPath | quote }}
           mountPropagation: Bidirectional
         - name: cdi
-          mountPath: /var/run/cdi
+          mountPath: {{ .Values.cdiRoot | quote }}
         - name: driver-root
           mountPath: /driver-root
           readOnly: true
@@ -220,7 +220,7 @@ spec:
         - name: NVIDIA_VISIBLE_DEVICES
           value: void
         - name: CDI_ROOT
-          value: /var/run/cdi
+          value: {{ .Values.cdiRoot | quote }}
         - name: NVIDIA_MIG_CONFIG_DEVICES
           value: all
         - name: NODE_NAME
@@ -259,7 +259,7 @@ spec:
           mountPath: {{ .Values.kubeletPlugin.kubeletPluginsDirectoryPath | quote }}
           mountPropagation: Bidirectional
         - name: cdi
-          mountPath: /var/run/cdi
+          mountPath: {{ .Values.cdiRoot | quote }}
         - name: driver-root
           mountPath: /driver-root
           readOnly: true
@@ -274,7 +274,7 @@ spec:
           path: {{ .Values.kubeletPlugin.kubeletPluginsDirectoryPath | quote }}
       - name: cdi
         hostPath:
-          path: /var/run/cdi
+          path: {{ .Values.cdiRoot | quote }}
       - name: driver-root-parent
         hostPath:
           # If nvidiaDriverRoot == "/" then its parent is itself. Otherwise, get

deployments/helm/nvidia-dra-driver-gpu/values.yaml

@@ -26,6 +26,10 @@ nvidiaDriverRoot: /
 # If not specified, the default path inferred from the nvidia-container-toolkit library version will be used.
 nvidiaCDIHookPath: ""
 
+# CDI root directory path.
+# This is where CDI spec files are stored and accessed by the runtime.
+cdiRoot: "/var/run/cdi"
+
 nameOverride: ""
 fullnameOverride: ""
 namespaceOverride: ""

hack/kubelet-plugin-prestart.sh

@@ -46,6 +46,7 @@ validate_and_exit_on_success () {
             /driver-root/bin \
             /driver-root/sbin \
             /driver-root/usr/bin \
+            /driver-root/usr/local/bin \
             /driver-root/sbin \
         -maxdepth 1 -type f -name "nvidia-smi" 2> /dev/null | head -n1
     )
@@ -59,6 +60,7 @@ validate_and_exit_on_success () {
             /driver-root/usr/lib64 \
             /driver-root/usr/lib/x86_64-linux-gnu \
             /driver-root/usr/lib/aarch64-linux-gnu \
+            /driver-root/usr/local/glibc/usr/lib \
             /driver-root/lib64 \
             /driver-root/lib/x86_64-linux-gnu \
             /driver-root/lib/aarch64-linux-gnu \