diff --git a/doc b/doc index b8279ce703..2b1a866f3a 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit b8279ce70372362f94c2b26fa4d0f60a9f00344c +Subproject commit 2b1a866f3a4bc3dc457b81b11a5e42f7cddba7a6 diff --git a/tests/conftest.py b/tests/conftest.py index 5250e7b5be..f6f7492395 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,6 +21,9 @@ from PIL import Image from syrupy.extensions.image import PNGImageSnapshotExtension +# Import GPU info module +from tests.gpu_info import detect_gpu_backend, GpuBackend + # Mock tkinter module for backward compatibility because it is a hard dependency for old Genesis versions has_tkinter = False try: @@ -107,12 +110,14 @@ def _skip_reason(reason): def is_mem_monitoring_supported(): - try: - assert sys.platform.startswith("linux") - subprocess.check_output(["nvidia-smi"], stderr=subprocess.STDOUT, timeout=10) + if not sys.platform.startswith("linux"): + return False, "mem-monitoring only supported on linux" + + backend = detect_gpu_backend() + if backend is not None: return True, None - except Exception as exc: # platform or nvidia-smi unavailable - return False, exc + + return False, "no supported GPU backend detected" def pytest_make_parametrize_id(config, val, argname): @@ -224,15 +229,17 @@ def _get_gpu_indices(): return tuple(map(int, cuda_visible_devices.split(","))) if sys.platform == "linux": - nvidia_gpu_interface_path = "/proc/driver/nvidia/gpus/" - try: - return tuple(range(len(os.listdir(nvidia_gpu_interface_path)))) - except FileNotFoundError: - warnings.warn( - f"'{nvidia_gpu_interface_path}' is not available. Multi-GPU support will be disabled. This is expected " - "on WSL2 where the NVIDIA proc interface is not mounted.", - stacklevel=2, - ) + backend = detect_gpu_backend() + if backend is not None: + device_count = backend.get_device_count() + if device_count > 0: + return tuple(range(device_count)) + + warnings.warn( + "No GPU backend detected. Multi-GPU support will be disabled. This is expected " + "on WSL2 where the GPU interface is not mounted.", + stacklevel=2, + ) return (0,) @@ -247,19 +254,15 @@ def _torch_get_gpu_idx(device): device_property = torch.cuda.get_device_properties(device) device_uuid = str(device_property.uuid) - nvidia_gpu_interface_path = "/proc/driver/nvidia/gpus/" - try: - for device_idx, device_path in enumerate(os.listdir(nvidia_gpu_interface_path)): - with open(os.path.join(nvidia_gpu_interface_path, device_path, "information"), "r") as f: - device_info = f.read() - if re.search(rf"GPU UUID:\s+GPU-{device_uuid}", device_info): - return device_idx - except FileNotFoundError: - warnings.warn( - f"'{nvidia_gpu_interface_path}' is not available. Multi-GPU support will be disabled. This is expected " - "on WSL2 where the NVIDIA proc interface is not mounted.", - stacklevel=2, - ) + backend = detect_gpu_backend() + if backend is not None: + return backend.get_device_index_from_uuid(device_uuid) + + warnings.warn( + "No GPU backend detected. Multi-GPU support will be disabled. This is expected " + "on WSL2 where the GPU interface is not mounted.", + stacklevel=2, + ) return -1 @@ -318,31 +321,12 @@ def pytest_xdist_auto_num_workers(config): else: # Cannot rely on 'torch' because this would force loading devices before configuring CUDA device visibility devices_vram_memory = None - try: - result = subprocess.run( - ["nvidia-smi", "--query-gpu=memory.total", "--format=csv,noheader,nounits"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - devices_vram_memory = tuple(int(e.strip()) for e in result.stdout.splitlines()) - except (FileNotFoundError, subprocess.CalledProcessError): - try: - result = subprocess.run( - ["rocm-smi", "--showmeminfo", "vram", "-d", "0-255"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - check=True, - text=True, - ) - devices_vram_memory = tuple( - int(m.group(1)) for m in re.finditer(r"VRAM Total:\s+(\d+)\s*MiB", result.stdout) - ) - except (FileNotFoundError, subprocess.CalledProcessError): - pass - if devices_vram_memory is not None: - assert len(set(devices_vram_memory)) == 1, "Heterogeneous Nvidia GPU devices not supported." + backend = detect_gpu_backend() + if backend is not None: + devices_vram_memory = backend.get_device_vram_mib() + + if devices_vram_memory is not None and devices_vram_memory: + assert len(set(devices_vram_memory)) == 1, "Heterogeneous GPU devices not supported." num_gpus = len(devices_vram_memory) vram_memory = sum(devices_vram_memory) / 1024 else: diff --git a/tests/gpu_info.py b/tests/gpu_info.py new file mode 100644 index 0000000000..2aa684c475 --- /dev/null +++ b/tests/gpu_info.py @@ -0,0 +1,375 @@ +""" +Cross-vendor GPU information module. +Provides abstract interface for querying GPU device information. +""" + +from abc import ABC, abstractmethod +from pathlib import Path +import subprocess +import warnings +import re +import shutil + + +class GpuBackend(ABC): + """Abstract base class for GPU backend implementations.""" + + @abstractmethod + def get_device_count(self) -> int: + """Get the number of available GPU devices.""" + pass + + @abstractmethod + def get_device_vram_mib(self) -> tuple[int, ...]: + """Get VRAM in MiB for each GPU device.""" + pass + + @abstractmethod + def get_device_index_from_uuid(self, device_uuid: str) -> int: + """Get device index from UUID.""" + pass + + @abstractmethod + def get_per_process_vram_mib(self) -> dict[int, int]: + """Get per-process VRAM usage in MiB.""" + pass + + @classmethod + @abstractmethod + def is_available(cls) -> bool: + """Report if the backend is available""" + pass + + +class NvidiaBackend(GpuBackend): + """NVIDIA GPU backend implementation.""" + + NVIDIA_GPU_INTERFACE_PATH = Path("/proc/driver/nvidia/gpus/") + NVIDIA_SMI = "nvidia-smi" + + UUID_RE_STR = "-".join((r"(?:[A-Fa-f0-9\-]{" f"{nr}" "})" for nr in (8, 4, 4, 4, 12))) + + def __init__(self): + # allow to later warn the user something is off and per-process memory monitoring won't + # supported. + self.is_nvidia_gpu_interface_populated = self._check_nvidia_gpu_interface_populated() + + @classmethod + def _check_nvidia_gpu_interface_populated(cls): + return cls.NVIDIA_GPU_INTERFACE_PATH.is_dir() and bool(list(cls.NVIDIA_GPU_INTERFACE_PATH.iterdir())) + + @classmethod + def is_available(cls) -> bool: + nvidia_smi_avail = shutil.which(cls.NVIDIA_SMI) is not None + + if nvidia_smi_avail and not cls._check_nvidia_gpu_interface_populated(): + warnings.warn( + f"{cls.NVIDIA_GPU_INTERFACE_PATH!s} is not populated. Fallback to using nvidia-smi to discover GPUs. " + "If you're running your app in a container, beware that per-process monitoring fails as the nvidia driver " + "lacks support for process namespace." + ) + + return nvidia_smi_avail + + def get_device_count(self) -> int: + """Get NVIDIA GPU count from proc interface.""" + return ( + (self.NVIDIA_GPU_INTERFACE_PATH.is_dir() and len(list(self.NVIDIA_GPU_INTERFACE_PATH.iterdir()))) + or + # fallback to nvidia-smi + len(self._get_nvidia_smi_list_gpus()) + ) + + @staticmethod + def _parse_nvidia_smi_query_memory(output: str) -> tuple[int, ...]: + vram_values = [] + for line in output.splitlines(): + line = line.strip() + if line: + # Convert from MB to MiB (they're the same for this purpose) + vram_values.append(int(line)) + + return tuple(vram_values) + + @classmethod + def _call_nvidia_smi(cls, args: list[str] = []) -> str: + return subprocess.check_output( + [cls.NVIDIA_SMI] + args, + encoding="utf-8", + ) + + def get_device_vram_mib(self) -> tuple[int, ...]: + """Get VRAM in MiB for each NVIDIA GPU using nvidia-smi.""" + + output = self._call_nvidia_smi( + [ + "--query-gpu=memory.total", + "--format=csv,noheader,nounits", + ] + ) + + per_process_vram_mib = self._parse_nvidia_smi_query_memory(output) + + if not per_process_vram_mib and not self.is_nvidia_gpu_interface_populated: + warnings.warn( + "No memory usage per process returned by nvidia-smi. " + "Your configuration seems to be broken as Nvidia GPU /proc interface is not populated, " + "indicating an abnormal driver setup. " + "Querying per-process metrics , especially within a container or a Kuberneted pod, " + "is likely to return nothing (nvidia driver does not support process namespaces)." + ) + + return per_process_vram_mib + + @classmethod + def _parse_nvidia_smi_list_gpus(cls, output: str) -> dict[int, str]: + device_idx_to_uuid = {} + for l in output.splitlines(): + m = re.match( + rf"GPU (\d*): .*\(UUID: GPU-({cls.UUID_RE_STR})\)", + l, + ) + + assert m is not None, "nvidia-smi has changed its format" + + device_idx = int(m.group(1)) + device_uuid = m.group(2) + + device_idx_to_uuid[device_idx] = device_uuid + + return device_idx_to_uuid + + def _get_nvidia_smi_list_gpus(self) -> dict[int, str]: + # fallback to nvidia-smi + output = self._call_nvidia_smi(["--list-gpus"]) + + return self._parse_nvidia_smi_list_gpus(output) + + @classmethod + def _extract_uuid_from_nvidia_proc_information(cls, device_info: str) -> str | None: + m = re.match(rf"GPU UUID:\s+GPU-({cls.UUID_RE_STR})", device_info) + + return m.group(1) if m is not None else None + + def _get_proc_nvidia_list_gpus(self) -> dict[int, str]: + device_idx_to_uuid = {} + for device_idx, device_path in enumerate(self.NVIDIA_GPU_INTERFACE_PATH.iterdir()): + device_info_path = device_path / "information" + if not device_info_path.exists(): + continue + + with device_info_path.open() as f: + device_info = f.read() + + device_uuid = self._extract_uuid_from_nvidia_proc_information(device_info) + assert device_uuid is not None, "/proc/driver/nvidia/gpus//information has changed its format" + device_idx_to_uuid[device_idx] = device_uuid + + return device_idx_to_uuid + + def get_device_index_from_uuid(self, device_uuid: str) -> int: + """Get NVIDIA device index from UUID.""" + + for idx_to_uuid_fn in ( + self._get_proc_nvidia_list_gpus, + self._get_nvidia_smi_list_gpus, # fallback to nvidia-smi + ): + map_idx_to_uuid = idx_to_uuid_fn() + + device_idx = next( + (map_idx for map_idx, map_uuid in map_idx_to_uuid.items() if device_uuid == map_uuid), None + ) + + if device_idx is not None: + return device_idx + + return -1 + + def get_per_process_vram_mib(self) -> dict[int, int]: + """Get per-process VRAM usage for NVIDIA GPUs.""" + output = self._call_nvidia_smi() + + return self._parse_nvidia_smi_output(output) + + def _parse_nvidia_smi_output(self, output: str) -> dict[int, int]: + """Parse nvidia-smi output for per-process memory usage.""" + section = 0 + subsec = 0 + res = {} + for line in output.split("\n"): + if line.startswith("|==========="): + section += 1 + subsec = 0 + continue + if line.startswith("+-------"): + subsec += 1 + continue + if section == 2 and subsec == 0: + if "No running processes" in line: + continue + split_line = line.split() + if len(split_line) >= 5: + pid = int(split_line[4]) + mem = int(split_line[-2].split("MiB")[0]) + res[pid] = mem + return res + + +class AmdBackend(GpuBackend): + """AMD GPU backend implementation.""" + + KFD_SYSFS_PATH = Path("/sys/devices/virtual/kfd/kfd/topology") + ROCM_SMI = "rocm-smi" + + @classmethod + def is_available(cls) -> bool: + return cls.KFD_SYSFS_PATH.is_dir() and (shutil.which(cls.ROCM_SMI) is not None) + + @classmethod + def _call_rocm_smi(cls, args: list[str] = []) -> str: + return subprocess.check_output( + [cls.ROCM_SMI] + args, + encoding="utf-8", + ) + + def _get_kfd_gpu_nodes_properties(self) -> dict[int, dict[str, int]]: + """Get KFD GPU node properties.""" + kfd_sysfs_path_nodes = self.KFD_SYSFS_PATH / "nodes" + gpu_nodes_properties = {} + + if not kfd_sysfs_path_nodes.is_dir(): + return {} + + for node_path in kfd_sysfs_path_nodes.iterdir(): + with (node_path / "properties").open() as node_properties_f: + properties_str = node_properties_f.read() + node_props = self._parse_kfd_node_properties(properties_str) + + if node_props["cpu_cores_count"] == 0: + gpu_nodes_properties[int(node_path.name)] = node_props + + return gpu_nodes_properties + + @staticmethod + def _parse_kfd_node_properties(kfd_properties_str: str) -> dict[str, int]: + """Parse KFD node properties string.""" + props = {} + for line in kfd_properties_str.split("\n"): + line = line.strip() + if not line: + continue + name, value_str = line.split() + props[name] = int(value_str) + + return props + + def get_device_count(self) -> int: + """Get AMD GPU count from KFD sysfs.""" + gpu_nodes = self._get_kfd_gpu_nodes_properties() + return len(gpu_nodes) + + @staticmethod + def _parse_rocm_smi_showmeminfo(output: str) -> tuple[int, ...]: + return tuple(int(m.group(1)) for m in re.finditer(r"VRAM Total:\s+(\d+)\s*MiB", output)) + + def get_device_vram_mib(self) -> tuple[int, ...]: + """Get VRAM in MiB for each AMD GPU using rocm-smi.""" + output = self._call_rocm_smi(["--showmeminfo", "vram", "-d", "0-255"]) + + # Parse rocm-smi output for VRAM info + return self._parse_rocm_smi_showmeminfo(output) + + def get_device_index_from_uuid(self, device_uuid: str) -> int: + """Get AMD device index from UUID.""" + device_uuid = device_uuid.replace("-", "") + hip_uuid = "".join([chr(int(device_uuid[i : i + 2], 16)) for i in range(0, len(device_uuid), 2)]) + unique_id = int(hip_uuid, 16) + + gpu_nodes_properties = self._get_kfd_gpu_nodes_properties() + + for node_rank, gpu_props in enumerate(gpu_nodes_properties.values()): + if gpu_props.get("unique_id") == unique_id: + return node_rank + + return -1 + + def get_per_process_vram_mib(self) -> dict[int, int]: + """Get per-process VRAM usage for AMD GPUs.""" + output = self._call_rocm_smi(["--showpids"]) + + return self._parse_rocm_smi_showpids(output) + + @staticmethod + def _parse_rocm_smi_showpids(output: str) -> dict[int, int]: + """Parse rocm-smi output for per-process memory usage.""" + NO_PIDS = "No KFD PIDs currently running" + + if NO_PIDS in output: + return {} + + PID = "PID" + VRAM_USED = "VRAM USED" + TABLE_BORDER_STR = "====" + + pid_idx = output.find(PID) + if pid_idx == -1: + return {} + + output = output[pid_idx:] + lines = output.split("\n") + if not lines: + return {} + + header = lines[0] + + # Extract column spans from header + header_to_span = {} + for header_m in re.finditer(r"(\S+( |$))+ *", header): + header_name = header_m.group(0).strip() + header_to_span[header_name] = header_m.span() + + if PID not in header_to_span or VRAM_USED not in header_to_span: + return {} + + pid_to_mem_use = {} + for line in lines[1:]: + line = line.strip() + if not line or line.startswith(TABLE_BORDER_STR): + continue + + header_to_info_str = {h: line[start:end].strip() for h, (start, end) in header_to_span.items()} + + pid = int(header_to_info_str[PID]) + mem_bytes = int(header_to_info_str[VRAM_USED]) + # Convert bytes to MiB + pid_to_mem_use[pid] = mem_bytes >> 20 + + return pid_to_mem_use + + +def detect_gpu_backend() -> GpuBackend | None: + """ + Detect available GPU backend. + + Returns: + NvidiaBackend if NVIDIA GPUs are detected, + AmdBackend if AMD GPUs are detected, + None otherwise. + """ + + Backend_candidates = [ + NvidiaBackend, + AmdBackend, + ] + + available_Backends = [Backend for Backend in Backend_candidates if Backend.is_available()] + + if not available_Backends: + return None + + if len(available_Backends) > 2: + warnings.warn("Multiple backends were detected on the current system.") + + Backend = available_Backends[0] + + return Backend() diff --git a/tests/monitor_test_mem.py b/tests/monitor_test_mem.py index 2c98dc233d..3e814f39ac 100644 --- a/tests/monitor_test_mem.py +++ b/tests/monitor_test_mem.py @@ -47,27 +47,17 @@ def parse_test_name(test_name: str) -> dict[str, str]: return filtered_params +# Import GPU info module +from tests.gpu_info import detect_gpu_backend + + def get_cuda_usage() -> dict[int, int]: - output = subprocess.check_output(["nvidia-smi"]).decode("utf-8") - section = 0 - subsec = 0 - res = {} - for line in output.split("\n"): - if line.startswith("|============"): - section += 1 - subsec = 0 - continue - if line.startswith("+-------"): - subsec += 1 - continue - if section == 2 and subsec == 0: - if "No running processes" in line: - continue - split_line = line.split() - pid = int(split_line[4]) - mem = int(split_line[-2].split("MiB")[0]) - res[pid] = mem - return res + """Get per-process GPU memory usage using the detected GPU backend.""" + backend = detect_gpu_backend() + if backend is not None: + return backend.get_per_process_vram_mib() + + raise RuntimeError("No supported GPU backend available.") def get_test_name_by_pid() -> dict[int, str]: diff --git a/tests/test_testing_utils.py b/tests/test_testing_utils.py new file mode 100644 index 0000000000..a477a20a7a --- /dev/null +++ b/tests/test_testing_utils.py @@ -0,0 +1,200 @@ +import pytest +from .gpu_info import NvidiaBackend, AmdBackend + + +# @pytest.mark.parametrize( +# "cli_output,expected_results", +# [], +# ) +def test_amd_backend_rocm_smi_parsing(): + """Test AMD backend rocm-smi parsing functionality.""" + + output = """ + + + ============================ ROCm System Management Interface ============================ + ===================================== KFD Processes ====================================== + KFD process information: + PID PROCESS NAME GPU(s) VRAM USED SDMA USED CU OCCUPANCY + 19191 [pytest-xdist r 1 2680758272 0 0 + 19188 [pytest-xdist r 1 2680078336 0 0 + 19206 [pytest-xdist r 1 2867257344 0 0 + 19194 [pytest-xdist r 1 2680664064 0 0 + 19200 [pytest-xdist r 1 2866966528 0 0 + 19209 [pytest-xdist r 1 2680922112 0 0 + 19197 [pytest-xdist r 1 2686627840 0 0 + 19203 [pytest-xdist r 1 2681561088 0 0 + ========================================================================================== + ================================== End of ROCm SMI Log =================================== + """ + + expected_output = { + 19191: 2556, + 19188: 2555, + 19206: 2734, + 19194: 2556, + 19200: 2734, + 19209: 2556, + 19197: 2562, + 19203: 2557, + } + results = AmdBackend._parse_rocm_smi_showpids(output) + assert results == expected_output + + +@pytest.mark.parametrize( + "cli_output,expected_results", + [ + ( + """ ++-----------------------------------------------------------------------------------------+ +| Processes: | +| GPU GI CI PID Type Process name GPU Memory | +| ID ID Usage | +|=========================================================================================| +| | ++-----------------------------------------------------------------------------------------+ +""", + {}, + ), + ( + """|===========| ++------+------+ +| Processes: GPU Memory | +| GPU GI CI PID Type Process name Usage | +| ID ID | MiB | +|=============|=======================================================|============| +| 0 0 0 1234 C python 100 | +| 0 0 0 5678 C python 200 | ++------+------+""", + { + 1234: 100, + 5678: 200, + }, + ), + ], +) +def test_nvidia_backend_nvidia_smi_parsing(cli_output, expected_results): + """Test NVIDIA backend nvidia-smi parsing functionality.""" + # Sample nvidia-smi output with process information + # Create NVIDIA backend and test its parsing method + nvidia_backend = NvidiaBackend() + results = nvidia_backend._parse_nvidia_smi_output(cli_output) + assert results == expected_results + + +# @pytest.mark.parametrize( +# "cli_output,expected_results", +# [], +# ) +def test_nvidia_backend_parse_nvidia_smi_query_memory(): + """Test NVIDIA backend _parse_nvidia_smi_query_memory method.""" + # Sample nvidia-smi query memory output + output = """12345 +67890 +45678 +""" + + expected_output = (12345, 67890, 45678) + + # Test the parsing method + results = NvidiaBackend._parse_nvidia_smi_query_memory(output) + assert results == expected_output + + +@pytest.mark.parametrize( + "cli_output,expected_results", + [ + ( + """GPU 0: Tesla V100-SXM2-32GB (UUID: GPU-12345678-90ab-cdef-1234-567890abcdef) +GPU 1: Tesla V100-SXM2-32GB (UUID: GPU-87654321-fedc-ba09-8765-4321fedcba98) +""", + {0: "12345678-90ab-cdef-1234-567890abcdef", 1: "87654321-fedc-ba09-8765-4321fedcba98"}, + ), + ( + """GPU 0: NVIDIA RTX PRO 6000 Blackwell Server Edition (UUID: GPU-517fb8f6-8dca-67f2-18f9-e99c9ff4159f)""", + {0: "517fb8f6-8dca-67f2-18f9-e99c9ff4159f"}, + ), + ], +) +def test_nvidia_backend_parse_nvidia_smi_list_gpus(cli_output, expected_results): + """Test NVIDIA backend _parse_nvidia_smi_list_gpus method.""" + # Sample nvidia-smi list-gpus output + # Test the parsing method + results = NvidiaBackend._parse_nvidia_smi_list_gpus(cli_output) + assert results == expected_results + + +def test_nvidia_backend_extract_uuid_from_nvidia_proc_information(): + """Test NVIDIA backend _extract_uuid_from_nvidia_proc_information method.""" + # Sample device information content + device_info = """GPU UUID: GPU-12345678-90ab-cdef-1234-567890abcdef""" + + expected_output = "12345678-90ab-cdef-1234-567890abcdef" + + # Test the extraction method + results = NvidiaBackend._extract_uuid_from_nvidia_proc_information(device_info) + assert results == expected_output + + +# @pytest.mark.parametrize( +# "cli_output,expected_results", +# [], +# ) +def test_amd_backend_parse_kfd_node_properties(): + """Test AMD backend _parse_kfd_node_properties method.""" + # Sample KFD node properties string + properties_str = """cpu_cores_count 0 +gpu_id 0 +max_waves_per_simd 10 +simd_count 4 +""" + + expected_output = {"cpu_cores_count": 0, "gpu_id": 0, "max_waves_per_simd": 10, "simd_count": 4} + + # Create AMD backend and test its parsing method + results = AmdBackend._parse_kfd_node_properties(properties_str) + assert results == expected_output + + +# @pytest.mark.parametrize( +# "cli_output,expected_results", +# [], +# ) +def test_amd_backend_parse_rocm_smi_showmeminfo(): + """Test AMD backend _parse_rocm_smi_showmeminfo method.""" + # Sample rocm-smi showmeminfo output + output = """VRAM Total: 16384 MiB +VRAM Total: 16384 MiB +""" + + expected_output = (16384, 16384) + + # Test the parsing method + results = AmdBackend._parse_rocm_smi_showmeminfo(output) + assert results == expected_output + + +# 97887 +# Sun Apr 12 06:10:31 2026 +# """ +# +-----------------------------------------------------------------------------------------+ +# | NVIDIA-SMI 580.126.09 Driver Version: 580.126.09 CUDA Version: 13.0 | +# +-----------------------------------------+------------------------+----------------------+ +# | GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC | +# | Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. | +# | | | MIG M. | +# |=========================================+========================+======================| +# | 0 NVIDIA RTX PRO 6000 Blac... Off | 00000000:61:00.0 Off | 0 | +# | N/A 24C P8 28W / 600W | 0MiB / 97887MiB | 0% Default | +# | | | Disabled | +# +-----------------------------------------+------------------------+----------------------+ + +# +-----------------------------------------------------------------------------------------+ +# | Processes: | +# | GPU GI CI PID Type Process name GPU Memory | +# | ID ID Usage | +# |=========================================================================================| +# | No running processes found | +# +-----------------------------------------------------------------------------------------+ +# """ diff --git a/tests/test_utils.py b/tests/test_utils.py index f66b527011..d84344ad46 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -17,6 +17,8 @@ from genesis.utils.urdf import compose_inertial_properties from .utils import assert_allclose +from . import monitor_test_mem +from .gpu_info import NvidiaBackend, AmdBackend TOL = 1e-7