chore: address review comments

Signed-off-by: Ajay Mishra <ajmishra@nvidia.com>

Author: XRFXLP

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/files/dcgmerrorsmapping.csv

@@ -0,0 +1,121 @@
+DCGM_FR_UNKNOWN,CONTACT_SUPPORT
+DCGM_FR_UNRECOGNIZED,CONTACT_SUPPORT
+DCGM_FR_PCI_REPLAY_RATE,CONTACT_SUPPORT
+DCGM_FR_VOLATILE_DBE_DETECTED,COMPONENT_RESET
+DCGM_FR_VOLATILE_SBE_DETECTED,NONE
+DCGM_FR_PENDING_PAGE_RETIREMENTS,NONE
+DCGM_FR_RETIRED_PAGES_LIMIT,CONTACT_SUPPORT
+DCGM_FR_RETIRED_PAGES_DBE_LIMIT,CONTACT_SUPPORT
+DCGM_FR_CORRUPT_INFOROM,COMPONENT_RESET
+DCGM_FR_CLOCKS_EVENT_THERMAL,CONTACT_SUPPORT
+DCGM_FR_CLOCK_THROTTLE_THERMAL,NONE
+DCGM_FR_POWER_UNREADABLE,RESTART_VM
+DCGM_FR_CLOCKS_EVENT_POWER,NONE
+DCGM_FR_CLOCK_THROTTLE_POWER,NONE
+DCGM_FR_NVLINK_ERROR_THRESHOLD,NONE
+DCGM_FR_NVLINK_DOWN,RESTART_VM
+DCGM_FR_NVSWITCH_FATAL_ERROR,CONTACT_SUPPORT
+DCGM_FR_NVSWITCH_NON_FATAL_ERROR,NONE
+DCGM_FR_NVSWITCH_DOWN,COMPONENT_RESET
+DCGM_FR_NO_ACCESS_TO_FILE,CONTACT_SUPPORT
+DCGM_FR_NVML_API,CONTACT_SUPPORT
+DCGM_FR_DEVICE_COUNT_MISMATCH,CONTACT_SUPPORT
+DCGM_FR_BAD_PARAMETER,CONTACT_SUPPORT
+DCGM_FR_CANNOT_OPEN_LIB,CONTACT_SUPPORT
+DCGM_FR_DENYLISTED_DRIVER,CONTACT_SUPPORT
+DCGM_FR_NVML_LIB_BAD,CONTACT_SUPPORT
+DCGM_FR_GRAPHICS_PROCESSES,CONTACT_SUPPORT
+DCGM_FR_HOSTENGINE_CONN,CONTACT_SUPPORT
+DCGM_FR_FIELD_QUERY,RESTART_VM
+DCGM_FR_BAD_CUDA_ENV,CONTACT_SUPPORT
+DCGM_FR_PERSISTENCE_MODE,CONTACT_SUPPORT
+DCGM_FR_LOW_BANDWIDTH,CONTACT_SUPPORT
+DCGM_FR_HIGH_LATENCY,CONTACT_SUPPORT
+DCGM_FR_CANNOT_GET_FIELD_TAG,CONTACT_SUPPORT
+DCGM_FR_FIELD_VIOLATION,RESTART_VM
+DCGM_FR_FIELD_THRESHOLD,RESTART_VM
+DCGM_FR_FIELD_VIOLATION_DBL,RESTART_VM
+DCGM_FR_FIELD_THRESHOLD_DBL,RESTART_VM
+DCGM_FR_UNSUPPORTED_FIELD_TYPE,RESTART_VM
+DCGM_FR_FIELD_THRESHOLD_TS,RESTART_VM
+DCGM_FR_FIELD_THRESHOLD_TS_DBL,RESTART_VM
+DCGM_FR_THERMAL_VIOLATIONS,CONTACT_SUPPORT
+DCGM_FR_THERMAL_VIOLATIONS_TS,CONTACT_SUPPORT
+DCGM_FR_TEMP_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_CLOCKS_EVENT_VIOLATION,RESTART_VM
+DCGM_FR_THROTTLING_VIOLATION,RESTART_VM
+DCGM_FR_INTERNAL,CONTACT_SUPPORT
+DCGM_FR_PCIE_GENERATION,CONTACT_SUPPORT
+DCGM_FR_PCIE_WIDTH,CONTACT_SUPPORT
+DCGM_FR_ABORTED,NONE
+DCGM_FR_TEST_DISABLED,NONE
+DCGM_FR_CANNOT_GET_STAT,CONTACT_SUPPORT
+DCGM_FR_STRESS_LEVEL,RESTART_VM
+DCGM_FR_CUDA_API,CONTACT_SUPPORT
+DCGM_FR_FAULTY_MEMORY,CONTACT_SUPPORT
+DCGM_FR_CANNOT_SET_WATCHES,RESTART_VM
+DCGM_FR_CUDA_UNBOUND,RESTART_VM
+DCGM_FR_ECC_DISABLED,CONTACT_SUPPORT
+DCGM_FR_MEMORY_ALLOC,RESTART_VM
+DCGM_FR_CUDA_DBE,CONTACT_SUPPORT
+DCGM_FR_MEMORY_MISMATCH,CONTACT_SUPPORT
+DCGM_FR_CUDA_DEVICE,CONTACT_SUPPORT
+DCGM_FR_ECC_UNSUPPORTED,CONTACT_SUPPORT
+DCGM_FR_ECC_PENDING,RESTART_VM
+DCGM_FR_MEMORY_BANDWIDTH,RESTART_VM
+DCGM_FR_TARGET_POWER,NONE
+DCGM_FR_API_FAIL,RESTART_VM
+DCGM_FR_API_FAIL_GPU,RESTART_VM
+DCGM_FR_CUDA_CONTEXT,CONTACT_SUPPORT
+DCGM_FR_DCGM_API,CONTACT_SUPPORT
+DCGM_FR_CONCURRENT_GPUS,CONTACT_SUPPORT
+DCGM_FR_TOO_MANY_ERRORS,CONTACT_SUPPORT
+DCGM_FR_NVLINK_CRC_ERROR_THRESHOLD,CONTACT_SUPPORT
+DCGM_FR_NVLINK_ERROR_CRITICAL,CONTACT_SUPPORT
+DCGM_FR_ENFORCED_POWER_LIMIT,CONTACT_SUPPORT
+DCGM_FR_MEMORY_ALLOC_HOST,RESTART_VM
+DCGM_FR_GPU_OP_MODE,CONTACT_SUPPORT
+DCGM_FR_NO_MEMORY_CLOCKS,CONTACT_SUPPORT
+DCGM_FR_NO_GRAPHICS_CLOCKS,NONE
+DCGM_FR_HAD_TO_RESTORE_STATE,RESTART_VM
+DCGM_FR_L1TAG_UNSUPPORTED,CONTACT_SUPPORT
+DCGM_FR_L1TAG_MISCOMPARE,CONTACT_SUPPORT
+DCGM_FR_ROW_REMAP_FAILURE,CONTACT_SUPPORT
+DCGM_FR_UNCONTAINED_ERROR,RESTART_VM
+DCGM_FR_EMPTY_GPU_LIST,CONTACT_SUPPORT
+DCGM_FR_DBE_PENDING_PAGE_RETIREMENTS,RESTART_VM
+DCGM_FR_UNCORRECTABLE_ROW_REMAP,NONE
+DCGM_FR_PENDING_ROW_REMAP,COMPONENT_RESET
+DCGM_FR_BROKEN_P2P_MEMORY_DEVICE,CONTACT_SUPPORT
+DCGM_FR_BROKEN_P2P_WRITER_DEVICE,CONTACT_SUPPORT
+DCGM_FR_NVSWITCH_NVLINK_DOWN,CONTACT_SUPPORT
+DCGM_FR_EUD_BINARY_PERMISSIONS,CONTACT_SUPPORT
+DCGM_FR_EUD_NON_ROOT_USER,CONTACT_SUPPORT
+DCGM_FR_EUD_SPAWN_FAILURE,CONTACT_SUPPORT
+DCGM_FR_EUD_TIMEOUT,CONTACT_SUPPORT
+DCGM_FR_EUD_ZOMBIE,CONTACT_SUPPORT
+DCGM_FR_EUD_NON_ZERO_EXIT_CODE,CONTACT_SUPPORT
+DCGM_FR_EUD_TEST_FAILED,CONTACT_SUPPORT
+DCGM_FR_FILE_CREATE_PERMISSIONS,CONTACT_SUPPORT
+DCGM_FR_PAUSE_RESUME_FAILED,CONTACT_SUPPORT
+DCGM_FR_PCIE_H_REPLAY_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_GPU_EXPECTED_NVLINKS_UP,CONTACT_SUPPORT
+DCGM_FR_NVSWITCH_EXPECTED_NVLINKS_UP,CONTACT_SUPPORT
+DCGM_FR_XID_ERROR,NONE
+DCGM_FR_SBE_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_DBE_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_PCIE_REPLAY_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_SBE_THRESHOLD_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_DBE_THRESHOLD_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_PCIE_REPLAY_THRESHOLD_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_CUDA_FM_NOT_INITIALIZED,CONTACT_SUPPORT
+DCGM_FR_SXID_ERROR,RESTART_VM
+DCGM_FR_GFLOPS_THRESHOLD_VIOLATION,CONTACT_SUPPORT
+DCGM_FR_NAN_VALUE,CONTACT_SUPPORT
+DCGM_FR_FABRIC_MANAGER_TRAINING_ERROR,CONTACT_SUPPORT
+DCGM_FR_BROKEN_P2P_PCIE_MEMORY_DEVICE,CONTACT_SUPPORT
+DCGM_FR_BROKEN_P2P_PCIE_WRITER_DEVICE,CONTACT_SUPPORT
+DCGM_FR_BROKEN_P2P_NVLINK_MEMORY_DEVICE,CONTACT_SUPPORT
+DCGM_FR_BROKEN_P2P_NVLINK_WRITER_DEVICE,CONTACT_SUPPORT
+DCGM_FR_TEST_SKIPPED,NONE
+DCGM_FR_ERROR_SENTINEL,NONE

distros/kubernetes/nvsentinel/charts/gpu-health-monitor/files/dcgmerrorsmapping.csv

@@ -32,4 +32,3 @@ data:
       processingStrategy: {{ include "preflight.processingStrategy" . | quote }}
     initContainers:
       {{- toYaml .Values.initContainers | nindent 6 }}
-{{ (.Files.Glob "files/dcgmerrorsmapping.csv").AsConfig | indent 2 }}

distros/kubernetes/nvsentinel/charts/preflight/files/dcgmerrorsmapping.csv

@@ -42,9 +42,6 @@ spec:
             - --port={{ .Values.webhook.port }}
             - --cert-dir=/certs
             - --config=/etc/preflight/config.yaml
-          env:
-            - name: PREFLIGHT_CONFIGMAP_NAME
-              value: {{ include "preflight.fullname" . }}-config
           ports:
             - name: https
               containerPort: {{ .Values.webhook.port }}

distros/kubernetes/nvsentinel/charts/preflight/templates/configmap.yaml

@@ -15,9 +15,9 @@
 replicaCount: 1
 
 image:
-  repository: ghcr.io/nvidia/nvsentinel/preflight
+  repository: xrfxlp/preflight
   pullPolicy: IfNotPresent
-  tag: "latest"
+  tag: 09550402
 
 imagePullSecrets: []
 nameOverride: ""
@@ -97,13 +97,10 @@ dcgm:
 
 initContainers:
   - name: preflight-dcgm-diag
-    image: ghcr.io/nvidia/nvsentinel/preflight-dcgm-diag:latest
+    image: xrfxlp/preflight-dcgm-diag:09550402
     volumeMounts:
       - name: nvsentinel-socket
         mountPath: /var/run
-      - name: dcgm-error-mapping
-        mountPath: /etc/dcgm
-        readOnly: true
     # Full corev1.Container fields supported:
     # resources:
     #   limits:

distros/kubernetes/nvsentinel/charts/preflight/templates/deployment.yaml

@@ -59,7 +59,10 @@ ENV PYTHONPATH=${PYTHONPATH} \
 COPY --from=build /app/dist/*.whl ./
 COPY --from=build /app/constraints.txt ./
 RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install ./dcgm_diag*.whl --constraint constraints.txt
+    pip install ./nvsentinel_dcgm_diag*.whl --constraint constraints.txt
+
+# Bundle error mapping file (static release data, not runtime config)
+COPY distros/kubernetes/nvsentinel/charts/gpu-health-monitor/files/dcgmerrorsmapping.csv /etc/dcgm/dcgmerrorsmapping.csv
 
 RUN groupadd -r nvsentinel && useradd -r -g nvsentinel nvsentinel
 USER nvsentinel

distros/kubernetes/nvsentinel/charts/preflight/values.yaml

@@ -69,46 +69,45 @@ def main() -> None:
 
     failures = [r for r in results if r.status == "fail"]
     warnings = [r for r in results if r.status == "warn"]
-
-    for r in results:
-        log.info(
-            "Test result",
-            extra={
-                "test": r.test_name,
-                "status": r.status,
-                "gpu": r.gpu_index,
-                "error_code": r.error_code,
-                "error": r.error_message,
-            },
-        )
+    passes = [r for r in results if r.status == "pass"]
 
     log.info(
         "Diagnostic summary",
         extra={
-            "passed": len(results) - len(failures) - len(warnings),
+            "passed": len(passes),
             "failed": len(failures),
             "warned": len(warnings),
+            "skipped": len(results) - len(passes) - len(failures) - len(warnings),
             "total": len(results),
         },
     )
 
-    for r in failures:
-        msg = f"{r.test_name} (GPU {r.gpu_index}): {r.error_message}"
-        log.error("DCGM diagnostic failed", extra={"gpu": r.gpu_uuid, "message": msg})
-        reporter.send_event(gpu_uuid=r.gpu_uuid, is_healthy=False, is_fatal=True, message=msg, error_code=r.error_code)
+    # Send one event per test result with specific test name
+    for r in results:
+        if r.status not in ("pass", "warn", "fail"):
+            continue
 
-    if failures:
-        sys.exit(1)
+        is_pass = r.status == "pass"
+        is_fatal = r.status == "fail"
+        message = "Test passed" if is_pass else r.error_message
 
-    for r in warnings:
-        msg = f"{r.test_name} (GPU {r.gpu_index}): {r.error_message}"
-        log.warning("DCGM diagnostic warning", extra={"gpu": r.gpu_uuid, "message": msg})
-        reporter.send_event(gpu_uuid=r.gpu_uuid, is_healthy=False, is_fatal=False, message=msg, error_code=r.error_code)
+        log.log(
+            logging.INFO if is_pass else (logging.ERROR if is_fatal else logging.WARNING),
+            f"Test {r.status}",
+            extra={"gpu": r.gpu_uuid, "test": r.test_name, "error_code": r.error_code, "detail": message},
+        )
+        reporter.send_event(
+            gpu_uuid=r.gpu_uuid,
+            is_healthy=is_pass,
+            is_fatal=is_fatal,
+            message=message,
+            error_code=r.error_code if not is_pass else 0,
+            test_name=r.test_name,
+        )
 
-    failed_gpus = {r.gpu_uuid for r in failures + warnings}
-    for uuid in diag.get_all_gpu_uuids():
-        if uuid not in failed_gpus:
-            reporter.send_event(gpu_uuid=uuid, is_healthy=True, is_fatal=False, message="DCGM diagnostic passed")
+    if failures:
+        log.error("DCGM diagnostic check failed")
+        sys.exit(1)
 
     log.info("DCGM diagnostic check passed")
     sys.exit(0)