Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmake/thirdparty/get_cucascade.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ function(find_and_configure_cucascade)
cuCascade 0.1.0
GLOBAL_TARGETS cuCascade::cucascade_topology_discovery
CPM_ARGS
GIT_REPOSITORY https://github.com/NVIDIA/cuCascade.git
GIT_TAG d515bb0536b8766bae61ec60a530df394467af64
GIT_REPOSITORY https://github.com/aminaramoon/cucs/
GIT_TAG no_rmm_in_discovery
OPTIONS "CUCASCADE_BUILD_TESTS OFF"
"CUCASCADE_BUILD_BENCHMARKS OFF"
"CUCASCADE_BUILD_SHARED_LIBS OFF"
Expand Down
38 changes: 37 additions & 1 deletion python/rapidsmpf/rapidsmpf/tests/test_rrun.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations

Expand Down Expand Up @@ -183,6 +183,42 @@ def body() -> None:

_run_in_subprocess(body)

def test_bind_does_not_initialize_cuda(self) -> None:
"""bind() must not latch CUDA while it temporarily clears CVD.

Topology discovery unsets CUDA_VISIBLE_DEVICES so cuCascade can see
physical GPU indices. If the CUDA driver initializes in that window,
the process permanently sees every GPU on the node even after CVD is
restored. On single-GPU CI that cannot be detected via UUID collision,
so we instead require that CUDA remain uninitialized after bind():
clearing CVD and then querying the device count must yield
cudaErrorNoDevice.
"""

def body() -> None:
import os

from cuda.bindings import runtime

from rapidsmpf.rrun.rrun import bind

# Restrict visibility, but do not touch CUDA before bind().
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
bind(gpu_id=0, cpu=True, memory=True, network=False, verify=False)

# If bind() left CUDA uninitialized, an empty CUDA_VISIBLE_DEVICES takes
# effect. If bind() latched "all devices" (or even just GPU 0), this change
# is ignored and cudaGetDeviceCount succeeds.
os.environ["CUDA_VISIBLE_DEVICES"] = ""
err, count = runtime.cudaGetDeviceCount()
assert err == runtime.cudaError_t.cudaErrorNoDevice, (
f"bind() initialized CUDA (cudaGetDeviceCount -> err={err}, "
f"count={count}); discovery must not touch the CUDA driver "
f"while CUDA_VISIBLE_DEVICES is temporarily cleared"
)

_run_in_subprocess(body)


class TestBindEffect:
"""Verify that bind() actually applies resource bindings.
Expand Down
Loading