fix: Run dcgm-exporter on GKE

Author: Ivaka

charts/kvisor/templates/_helpers.tpl

@@ -345,6 +345,13 @@ Only used as a fallback when controller.extraArgs.cloud-provider is not set.
 {{- dig "castai" "provider" "" (.Values.global | default dict) -}}
 {{- end }}
 
+{{/*
+Returns true if GPU metrics collection is enabled.
+*/}}
+{{- define "kvisor.gpuEnabled" -}}
+{{- dig "gpu" "enabled" false .Values.agent -}}
+{{- end }}
+
 {{/*
 Resolve CASTAI_API_GRPC_ADDR: global.castai.grpcURL > .Values.castai.grpcAddr
 */}}

charts/kvisor/templates/agent.yaml

@@ -56,7 +56,7 @@ spec:
       securityContext:
         {{- toYaml .Values.agent.securityContext | nindent 8 }}
       priorityClassName: {{ .Values.agent.priorityClass }}
-      {{- if (dig "gpu" "enabled" false .Values.agent) }}
+      {{- if (include "kvisor.gpuEnabled" .) }}
       initContainers:
         - name: gpu-feature-discovery
           image: "{{ .Values.image.repository }}-agent:{{ .Values.image.tag | default .Chart.AppVersion }}"
@@ -108,7 +108,7 @@ spec:
           {{- range $key, $value := .Values.agent.extraArgs }}
             - "--{{ $key }}={{ $value }}"
           {{- end }}
-          {{- if (dig "gpu" "enabled" false .Values.agent) }}
+          {{- if (include "kvisor.gpuEnabled" .) }}
             - "--gpu-enabled=true"
             - "--gpu-export-interval={{ .Values.agent.gpu.exportInterval }}"
             - "--gpu-dcgm-port={{ .Values.agent.gpu.dcgmExporter.port }}"
@@ -197,7 +197,7 @@ spec:
             {{- with .Values.agent.extraVolumeMounts }}
             {{- toYaml . | nindent 12 }}
             {{- end }}
-            {{- if (dig "gpu" "enabled" false .Values.agent) }}
+            {{- if (include "kvisor.gpuEnabled" .) }}
             - name: gpu-shared
               mountPath: {{ .Values.agent.gpu.sharedDir }}
             {{- end }}
@@ -307,11 +307,13 @@ spec:
               readOnly: true
       {{- end }}
       {{- end }}
-      {{- if (dig "gpu" "enabled" false .Values.agent) }}
+      {{- if (include "kvisor.gpuEnabled" .) }}
+        {{- $provider := include "kvisor.cloudProvider" . }}
         - name: dcgm-exporter
           image: "{{ .Values.agent.gpu.dcgmExporter.image.repository }}:{{ .Values.agent.gpu.dcgmExporter.image.tag }}"
           imagePullPolicy: {{ .Values.agent.gpu.dcgmExporter.image.pullPolicy }}
           securityContext:
+            privileged: {{ eq $provider "gke" }}
             runAsNonRoot: false
             runAsUser: 0
             allowPrivilegeEscalation: true
@@ -331,7 +333,32 @@ spec:
                 echo "Existing DCGM exporter found on node. Standing by."
                 while true; do sleep 3600; done
               fi
+              {{- if eq $provider "gke" }}
+              # Wait for GKE NVIDIA driver installation (cos-nvidia-installer runs asynchronously).
+              # 30s timeout handles CPU-only nodes where drivers will never appear.
+              elapsed=0
+              while [ $elapsed -lt 30 ]; do
+                if ls "/usr/local/nvidia/lib64/libnvidia-ml.so.1" &>/dev/null; then
+                  echo "nvml drivers found"
+                  break
+                fi
+                echo "waiting for nvml drivers ($elapsed/30s)"
+                sleep 1
+                elapsed=$((elapsed + 1))
+              done
+              if ! ls "/usr/local/nvidia/lib64/libnvidia-ml.so.1" &>/dev/null; then
+                echo "nvml drivers not found after 30s (CPU-only node). Standing by."
+                while true; do sleep 3600; done
+              fi
               hostname $NODE_NAME; exec dcgm-exporter -f /etc/dcgm-exporter/counters.csv
+              {{- else }}
+              hostname $NODE_NAME
+              for ((;;)); do
+                dcgm-exporter -f /etc/dcgm-exporter/counters.csv
+                echo "dcgm-exporter exited (no GPU or driver not ready). Retrying in 60s."
+                sleep 60
+              done
+              {{- end }}
           ports:
             - name: dcgm-metrics
               containerPort: {{ .Values.agent.gpu.dcgmExporter.port }}
@@ -349,6 +376,10 @@ spec:
               valueFrom:
                 fieldRef:
                   fieldPath: spec.nodeName
+            {{- if eq $provider "gke" }}
+            - name: DISABLE_STARTUP_VALIDATE
+              value: "true"
+            {{- end }}
           volumeMounts:
             - name: gpu-shared
               mountPath: {{ .Values.agent.gpu.sharedDir }}
@@ -358,6 +389,11 @@ spec:
             - name: dcgm-config
               mountPath: /etc/dcgm-exporter
               readOnly: true
+            {{- if eq $provider "gke" }}
+            - name: nvidia-install-dir-host
+              mountPath: /usr/local/nvidia
+              readOnly: true
+            {{- end }}
       {{- end }}
       dnsPolicy: {{.Values.agent.dnsPolicy}}
       {{- with .Values.agent.nodeSelector }}
@@ -412,7 +448,7 @@ spec:
                       fieldRef:
                         fieldPath: metadata.namespace
       {{- end }}
-      {{- if (dig "gpu" "enabled" false .Values.agent) }}
+      {{- if (include "kvisor.gpuEnabled" .) }}
         - name: gpu-shared
           emptyDir: {}
         - name: pod-gpu-resources
@@ -421,6 +457,12 @@ spec:
         - name: dcgm-config
           configMap:
             name: {{ include "kvisor.agent.fullname" . }}-dcgm-config
+        {{- if eq (include "kvisor.cloudProvider" .) "gke" }}
+        - name: nvidia-install-dir-host
+          hostPath:
+            path: /home/kubernetes/bin/nvidia
+            type: DirectoryOrCreate
+        {{- end }}
         - name: gpu-sa-token
           projected:
             sources:
@@ -473,7 +515,7 @@ rules:
       - get
       - list
       - watch
-  {{- if (dig "gpu" "enabled" false .Values.agent) }}
+  {{- if (include "kvisor.gpuEnabled" .) }}
   - apiGroups:
       - ""
     resources:

charts/kvisor/templates/controller.yaml

@@ -403,14 +403,6 @@ rules:
       - get
       - list
       - watch
-  - apiGroups:
-      - "argoproj.io"
-    resources:
-      - rollouts
-    verbs:
-      - get
-      - list
-      - watch
   - apiGroups:
       - "networking.k8s.io"
     resources:

charts/kvisor/templates/dcgm-configmap.yaml

@@ -1,4 +1,4 @@
-{{- if (dig "gpu" "enabled" false .Values.agent) }}
+{{- if (include "kvisor.gpuEnabled" .) }}
 apiVersion: v1
 kind: ConfigMap
 metadata:

charts/kvisor/values.yaml

@@ -189,9 +189,10 @@ agent:
       port: 9400
       resources:
         requests:
-          memory: 64Mi
+          cpu: 100m
+          memory: 128Mi
         limits:
-          memory: 256Mi
+          memory: 500Mi
 
   # Reliability metrics collection via OBI (OpenTelemetry eBPF Instrumentation).
   # Deploys OBI as a sidecar container in the kvisor agent DaemonSet to collect