Merge branch 'main' into k8s-ds-design-doc

Author: lalitadithya

.github/actions/publish-container/action.yml

@@ -32,6 +32,16 @@ inputs:
 runs:
   using: 'composite'
   steps:
+    - name: Workaround for freeing up more disk space
+      shell: bash
+      run: |
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /opt/ghc
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        sudo docker image prune --all --force
+        sudo docker system prune -f
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435  # v3.11.1
       with:
@@ -86,6 +96,22 @@ runs:
           rm -rf /tmp/.buildx-cache
           mv /tmp/.buildx-cache-new /tmp/.buildx-cache
         fi
+
+    - name: Free disk space for SBOM generation
+      shell: bash
+      run: |
+        echo "=== Disk usage BEFORE cleanup ==="
+        df -h
+        # Remove pre-installed tools (if they exist on this runner)
+        sudo rm -rf /usr/local/lib/android || true
+        sudo rm -rf /usr/share/dotnet || true
+        sudo rm -rf /opt/ghc || true
+        # Thorough Docker cleanup - build cache no longer needed after push
+        docker buildx prune -f --all || true
+        docker system prune -af --volumes || true
+        rm -rf /tmp/.buildx-cache || true
+        echo "=== Disk usage AFTER cleanup ==="
+        df -h
     
     - name: Generate SBOM and Attest
       uses: ./.github/actions/sbom-and-attest

.github/copy-pr-bot.yaml

@@ -25,3 +25,4 @@ additional_trustees:
   - deesharma24
   - natherz97
   - suket22
+  - cbumb

.github/workflows/cleanup-untagged-images.yml

@@ -89,7 +89,7 @@ jobs:
           - nvsentinel-janitor
           - nvsentinel-fake-dcgm
           - nvsentinel/preflight
-          - nvsentinel/ping
+          - nvsentinel/preflight-dcgm-diag
 
     steps:
       - name: Delete untagged images for ${{ matrix.package }}

.github/workflows/container-build-test.yml

@@ -65,6 +65,14 @@ jobs:
             make_command: 'make -C log-collector docker-build-log-collector'
           - component: file-server-cleanup
             make_command: 'make -C log-collector docker-build-file-server-cleanup'
+          # Preflight Checks (Docker-based)
+          - component: preflight-dcgm-diag
+            make_command: 'make -C preflight-checks/dcgm-diag docker-build'
+          - component: preflight-nccl-loopback
+            make_command: 'make -C preflight-checks/nccl-loopback docker-build'
+          # GPU Reset (Docker-based)
+          - component: gpu-reset
+            make_command: 'make -C gpu-reset docker-build'
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
@@ -128,8 +136,6 @@ jobs:
             path: .
           - module: preflight
             path: .
-          - module: preflight-checks/ping
-            path: .
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 

.github/workflows/lint-test.yml

@@ -58,6 +58,10 @@ jobs:
             make_command: 'make -C log-collector lint-log-collector'
             step_name: 'Run lint'
             replace_imports: 'false'
+          - component: gpu-reset
+            make_command: 'make -C gpu-reset lint'
+            step_name: 'Run lint'
+            replace_imports: 'false'
           - component: file-server-cleanup
             make_command: 'make -C log-collector lint-file-server-cleanup'
             step_name: 'Run lint'
@@ -139,7 +143,8 @@ jobs:
     strategy:
       matrix:
         component:
-          - ping
+          - dcgm-diag
+          - nccl-loopback
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
 
@@ -149,15 +154,6 @@ jobs:
       - name: Run lint and test
         run: make -C preflight-checks/${{ matrix.component }} lint-test
 
-      - name: Upload artifacts
-        uses: ./.github/actions/upload-test-artifacts
-        with:
-          component-name: preflight-${{ matrix.component }}
-          file-paths: |
-            preflight-checks/${{ matrix.component }}/coverage.xml
-            preflight-checks/${{ matrix.component }}/coverage.txt
-            preflight-checks/${{ matrix.component }}/report.xml
-
   modules-lint-test:
     if: github.repository == 'nvidia/nvsentinel'
     runs-on: linux-amd64-cpu16

.github/workflows/publish.yml

@@ -117,6 +117,15 @@ jobs:
           - component: file-server-cleanup
             make_command: 'make -C log-collector docker-publish-file-server-cleanup'
             container_name: 'nvsentinel/file-server-cleanup'
+          - component: preflight-dcgm-diag
+            make_command: 'make -C preflight-checks/dcgm-diag docker-publish'
+            container_name: 'nvsentinel/preflight-dcgm-diag'
+          - component: preflight-nccl-loopback
+            make_command: 'make -C preflight-checks/nccl-loopback docker-publish'
+            container_name: 'nvsentinel/preflight-nccl-loopback'
+          - component: gpu-reset
+            make_command: 'make -C gpu-reset docker-publish'
+            container_name: 'nvsentinel/gpu-reset'
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:

.gitignore

@@ -444,3 +444,5 @@ tests/scale-tests/FQM_LATENCY_TEST_PLAN.md
 tests/scale-tests/CONCURRENT_DRAIN_TEST_PLAN.md
 tests/scale-tests/results/*.csv
 tests/scale-tests/cmd/fqm-scale-test/results/
+preflight-checks/dcgm-diag/dcgm-diag
+preflight/preflight

Makefile

@@ -62,7 +62,8 @@ PYTHON_MODULES := \
 
 # Container-only modules
 CONTAINER_MODULES := \
-	log-collector
+	log-collector \
+    gpu-reset
 
 # Special modules requiring private repo access
 PRIVATE_MODULES := \
@@ -267,7 +268,7 @@ install-go-ci: ## Install Go $(GO_VERSION) for CI environments (Linux/macOS, amd
 
 # Lint and test all modules (delegates to sub-Makefiles)
 .PHONY: lint-test-all
-lint-test-all: protos-lint license-headers-lint gomod-lint health-monitors-lint-test-all go-lint-test-all python-lint-test-all kubernetes-distro-lint log-collector-lint ## Lint and test all modules
+lint-test-all: protos-lint license-headers-lint gomod-lint health-monitors-lint-test-all go-lint-test-all python-lint-test-all kubernetes-distro-lint log-collector-lint gpu-reset-lint ## Lint and test all modules
 
 # Health monitors lint-test (delegate to health-monitors/Makefile)
 .PHONY: health-monitors-lint-test-all
@@ -301,6 +302,8 @@ protos-generate: protos-clean ## Generate protobuf files from .proto sources
 	$(MAKE) -C api protos-generate
 	# Generate Python protobuf files for gpu-health-monitor
 	$(MAKE) -C health-monitors/gpu-health-monitor protos-generate
+	# Generate Python protobuf files for dcgm-diag preflight check
+	$(MAKE) -C preflight-checks/dcgm-diag protos-generate
 
 # Check protobuf files
 .PHONY: protos-lint
@@ -502,6 +505,12 @@ log-collector-lint: ## Lint shell scripts in log collector
 	@echo "Linting log collector shell scripts..."
 	$(MAKE) -C log-collector lint
 
+# GPU reset lint (shell script)
+.PHONY: gpu-reset-lint
+gpu-reset-lint: ## Lint shell scripts in GPU reset
+	@echo "Linting GPU reset shell scripts..."
+	$(MAKE) -C gpu-reset lint
+
 # Build targets (delegate to sub-Makefiles for better organization)
 .PHONY: build-all
 build-all: build-health-monitors build-main-modules ## Build all modules

commons/go.sum

@@ -80,8 +80,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
-github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
-github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI=
+github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

demos/local-fault-injection-demo/scripts/01-show-cluster.sh

@@ -104,14 +104,28 @@ main() {
         node_name=$(echo "$node" | cut -d'/' -f2)
         echo "Node: $node_name"
 
-        # Get all conditions
-        kubectl get "$node" -o jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}' | \
-            grep -v "^Ready\|^MemoryPressure\|^DiskPressure\|^PIDPressure\|^NetworkUnavailable" || true
+        # Get all conditions and capture for checking while also displaying
+        node_conditions=$(kubectl get "$node" -o jsonpath='{range .status.conditions[*]}{.type}{"\t"}{.status}{"\t"}{.message}{"\n"}{end}') || {
+            echo "  ⚠️  Failed to fetch conditions for $node_name"
+            continue
+        }
+        conditions_output=$(echo "$node_conditions" | \
+            grep -v "^Ready\|^MemoryPressure\|^DiskPressure\|^PIDPressure\|^NetworkUnavailable" || true)
+        
+        echo "$conditions_output"
+        
+        # Track if we found any actual health failures (Status=True indicating a problem)
+        if echo "$conditions_output" | grep -q $'\t'"True"$'\t'; then
+            has_conditions=true
+        fi
 
     done
 
-    if [ "$has_conditions" = false ]; then
-        echo "  ℹ️  No health event conditions found (cluster is healthy)"
+    echo ""
+    if [ "$has_conditions" = true ]; then
+        echo "  ⚠️  Health issues detected (see conditions above)"
+    else
+        echo "  ✅ No health event conditions found (cluster is healthy)"
     fi
 
     section "Recent Events"