Run nvidia-smi after modules are loaded in driver ds startup probe

This commit eliminates the race condition where the startup probe
in the driver daemonset runs after the kernel modules are built
(and installed) but before the modules are loaded into the kernel.
In this case, the invocation of nvidia-smi (by the startup probe)
is what is actually loading the nvidia kernel module and not the
modprobe we perform in our driver container scripts. As a result,
the nvidia driver will be loaded with a default configuration --
none of the custom kernel module parameters provided by users
(via a configmap) or set by our driver container will get applied.

Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>

Author: cdesiniotis

assets/state-driver/0500_daemonset.yaml

@@ -137,7 +137,7 @@ spec:
         startupProbe:
           exec:
             command:
-              [sh, -c, 'nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
+              [sh, -c, '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready']
           initialDelaySeconds: 60
           failureThreshold: 120
           successThreshold: 1

internal/state/testdata/golden/driver-additional-configs.yaml

@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-full-spec.yaml

@@ -182,7 +182,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-gdrcopy-openshift.yaml

@@ -238,7 +238,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-gdrcopy.yaml

@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-gds.yaml

@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-minimal.yaml

@@ -168,7 +168,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-openshift-drivertoolkit.yaml

@@ -238,7 +238,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-precompiled.yaml

@@ -170,7 +170,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10

internal/state/testdata/golden/driver-rdma-hostmofed.yaml

@@ -172,7 +172,7 @@ spec:
             command:
             - sh
             - -c
-            - nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready
+            - '[ -f /sys/module/nvidia/refcnt ] && nvidia-smi && touch /run/nvidia/validations/.driver-ctr-ready'
           failureThreshold: 120
           initialDelaySeconds: 60
           periodSeconds: 10