Run toolkit validation in operand init containers

The toolkit-validation init containers in operand DaemonSets previously
polled for a toolkit-ready sentinel file on the host. During driver
reinstall cycles, the driver-manager restores node scheduling labels
before the driver container has loaded the nvidia kernel module. All
operand pods get created simultaneously and find a stale toolkit-ready
file from a previous cycle, passing the init gate while nvidia-smi would
actually fail.

Replace the shell-based sentinel file check with a nvidia-validator
check. This runs nvidia-smi through the toolkit runtime wrapper and
retries until it succeeds, validating both toolkit injection and driver
module readiness without depending on host sentinel files.

Author: rajathagasthya

assets/gpu-feature-discovery/0500_daemonset.yaml

@@ -29,13 +29,20 @@ spec:
         - name: toolkit-validation
           image: "FILLED BY THE OPERATOR"
           command: ['sh', '-c']
-          args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"]
+          args: ["nvidia-validator"]
+          env:
+          - name: NVIDIA_VISIBLE_DEVICES
+            value: "all"
+          - name: WITH_WAIT
+            value: "true"
+          - name: COMPONENT
+            value: toolkit
           securityContext:
             privileged: true
           volumeMounts:
             - name: run-nvidia
               mountPath: /run/nvidia
-              mountPropagation: HostToContainer
+              mountPropagation: Bidirectional
         - name: config-manager-init
           image: "FILLED BY THE OPERATOR"
           command: ["config-manager"]

assets/state-container-toolkit/0500_daemonset.yaml

@@ -77,6 +77,10 @@ spec:
               fieldPath: metadata.namespace
         imagePullPolicy: IfNotPresent
         name: nvidia-container-toolkit-ctr
+        lifecycle:
+          preStop:
+            exec:
+              command: ["/bin/sh", "-c", "rm -f /run/nvidia/validations/toolkit-ready"]
         securityContext:
           privileged: true
           seLinuxOptions:

assets/state-dcgm-exporter/0800_daemonset.yaml

@@ -28,13 +28,20 @@ spec:
       - name: toolkit-validation
         image: "FILLED BY THE OPERATOR"
         command: ['sh', '-c']
-        args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"]
+        args: ["nvidia-validator"]
+        env:
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: "all"
+        - name: WITH_WAIT
+          value: "true"
+        - name: COMPONENT
+          value: toolkit
         securityContext:
           privileged: true
         volumeMounts:
           - name: run-nvidia
-            mountPath: "/run/nvidia"
-            mountPropagation: HostToContainer
+            mountPath: /run/nvidia
+            mountPropagation: Bidirectional
       containers:
       - image: "FILLED BY THE OPERATOR"
         name: nvidia-dcgm-exporter

assets/state-dcgm/0400_dcgm.yml

@@ -28,13 +28,20 @@ spec:
       - name: toolkit-validation
         image: "FILLED BY THE OPERATOR"
         command: ['sh', '-c']
-        args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"]
+        args: ["nvidia-validator"]
+        env:
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: "all"
+        - name: WITH_WAIT
+          value: "true"
+        - name: COMPONENT
+          value: toolkit
         securityContext:
           privileged: true
         volumeMounts:
           - name: run-nvidia
             mountPath: /run/nvidia
-            mountPropagation: HostToContainer
+            mountPropagation: Bidirectional
       containers:
       - image: "FILLED BY THE OPERATOR"
         name: nvidia-dcgm-ctr

assets/state-device-plugin/0500_daemonset.yaml

@@ -28,13 +28,20 @@ spec:
       - image: "FILLED BY THE OPERATOR"
         name: toolkit-validation
         command: ['sh', '-c']
-        args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"]
+        args: ["nvidia-validator"]
+        env:
+        - name: NVIDIA_VISIBLE_DEVICES
+          value: "all"
+        - name: WITH_WAIT
+          value: "true"
+        - name: COMPONENT
+          value: toolkit
         securityContext:
           privileged: true
         volumeMounts:
           - name: run-nvidia-validations
             mountPath: /run/nvidia/validations
-            mountPropagation: HostToContainer
+            mountPropagation: Bidirectional
       - image: "FILLED BY THE OPERATOR"
         name: config-manager-init
         command: ["config-manager"]

assets/state-driver/0500_daemonset.yaml

@@ -150,7 +150,7 @@ spec:
         lifecycle:
           preStop:
             exec:
-              command: ["/bin/sh", "-c", "rm -f /run/nvidia/validations/.driver-ctr-ready"]
+              command: ["/bin/sh", "-c", "rm -f /run/nvidia/validations/.driver-ctr-ready /run/nvidia/validations/driver-ready"]
       - image: "FILLED BY THE OPERATOR"
         imagePullPolicy: IfNotPresent
         name: nvidia-peermem-ctr

assets/state-mig-manager/0600_daemonset.yaml

@@ -28,13 +28,20 @@ spec:
         - name: toolkit-validation
           image: "FILLED BY THE OPERATOR"
           command: ['sh', '-c']
-          args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container toolkit to be setup; sleep 5; done"]
+          args: ["nvidia-validator"]
+          env:
+          - name: NVIDIA_VISIBLE_DEVICES
+            value: "all"
+          - name: WITH_WAIT
+            value: "true"
+          - name: COMPONENT
+            value: toolkit
           securityContext:
             privileged: true
           volumeMounts:
             - name: run-nvidia-validations
               mountPath: /run/nvidia/validations
-              mountPropagation: HostToContainer
+              mountPropagation: Bidirectional
       containers:
       - name: nvidia-mig-manager
         image: "FILLED BY THE OPERATOR"

assets/state-mps-control-daemon/0400_daemonset.yaml

@@ -30,13 +30,20 @@ spec:
         - image: "FILLED BY THE OPERATOR"
           name: toolkit-validation
           command: ['sh', '-c']
-          args: ["until [ -f /run/nvidia/validations/toolkit-ready ]; do echo waiting for nvidia container stack to be setup; sleep 5; done"]
+          args: ["nvidia-validator"]
+          env:
+          - name: NVIDIA_VISIBLE_DEVICES
+            value: "all"
+          - name: WITH_WAIT
+            value: "true"
+          - name: COMPONENT
+            value: toolkit
           securityContext:
             privileged: true
           volumeMounts:
             - name: run-nvidia
               mountPath: /run/nvidia
-              mountPropagation: HostToContainer
+              mountPropagation: Bidirectional
         - image: "FILLED BY THE OPERATOR"
           name: mps-control-daemon-mounts
           command: [mps-control-daemon, mount-shm]