Merge pull request #1939 from cdesiniotis/fix-driver-startup-probe

cdesiniotis · web-flow · commit 51dd7a28cd86 · 2025-11-26T10:14:10.000-08:00
Run nvidia-smi after modules are loaded in driver ds startup probe
diff --git a/assets/state-driver/0500_daemonset.yaml b/assets/state-driver/0500_daemonset.yaml
@@ -137,7 +137,7 @@ spec:
         startupProbe:
           exec:
             command:
-              [sh, -c, 'nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
+              [sh, -c, '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
           initialDelaySeconds: 60
           failureThreshold: 120
           successThreshold: 1
diff --git a/internal/state/testdata/golden/driver-additional-configs.yaml b/internal/state/testdata/golden/driver-additional-configs.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-full-spec.yaml b/internal/state/testdata/golden/driver-full-spec.yaml
@@ -182,7 +182,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml b/internal/state/testdata/golden/driver-gdrcopy-openshift.yaml
@@ -238,7 +238,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-gdrcopy.yaml b/internal/state/testdata/golden/driver-gdrcopy.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-gds.yaml b/internal/state/testdata/golden/driver-gds.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-minimal.yaml b/internal/state/testdata/golden/driver-minimal.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml b/internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml
@@ -238,7 +238,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-precompiled.yaml b/internal/state/testdata/golden/driver-precompiled.yaml
@@ -170,7 +170,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-rdma-hostmofed.yaml b/internal/state/testdata/golden/driver-rdma-hostmofed.yaml
@@ -172,7 +172,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-rdma.yaml b/internal/state/testdata/golden/driver-rdma.yaml
@@ -170,7 +170,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-secret-env.yaml b/internal/state/testdata/golden/driver-secret-env.yaml
@@ -171,7 +171,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml b/internal/state/testdata/golden/driver-vgpu-licensing-secret.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/internal/state/testdata/golden/driver-vgpu-licensing.yaml b/internal/state/testdata/golden/driver-vgpu-licensing.yaml
@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10
diff --git a/manifests/state-driver/0500_daemonset.yaml b/manifests/state-driver/0500_daemonset.yaml
@@ -331,7 +331,7 @@ spec:
         startupProbe:
           exec:
             command:
-              [sh, -c, 'nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
+              [sh, -c, '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
           initialDelaySeconds: {{ .Driver.Spec.StartupProbe.InitialDelaySeconds }}
           failureThreshold: {{ .Driver.Spec.StartupProbe.FailureThreshold }}
           successThreshold: {{ .Driver.Spec.StartupProbe.SuccessThreshold }}
