Use pytest-xdist to parallelize tests

ci/run_pytests.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
-# Copyright (c) 2024, NVIDIA CORPORATION.
+# Copyright (c) 2024-2025, NVIDIA CORPORATION.
 
 set -euo pipefail
 
+# It is essential to cd into python/rmm/rmm as `pytest-xdist` + `coverage` seem to work only at this directory level.
+
 # Support invoking run_pytests.sh outside the script directory
-cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/rmm/
+cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../python/rmm/rmm/
 
-pytest --cache-clear -v "$@" .
+pytest --cache-clear -v "$@" tests

ci/test_python.sh

@@ -42,6 +42,8 @@ rapids-logger "pytest rmm"
 
 ./ci/run_pytests.sh \
     --junitxml="${RAPIDS_TESTS_DIR}/junit-rmm.xml" \
+    --numprocesses=8 \
+    --dist=worksteal \
     --cov-config=.coveragerc \
     --cov=rmm \
     --cov-report=xml:"${RAPIDS_COVERAGE_DIR}/rmm-coverage.xml" \

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -31,6 +31,7 @@ dependencies:
 - pre-commit
 - pytest
 - pytest-cov
+- pytest-xdist
 - python>=3.10,<3.14
 - rapids-build-backend>=0.4.0,<0.5.0.dev0
 - rapids-logger==0.2.*,>=0.0.0a0

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -31,6 +31,7 @@ dependencies:
 - pre-commit
 - pytest
 - pytest-cov
+- pytest-xdist
 - python>=3.10,<3.14
 - rapids-build-backend>=0.4.0,<0.5.0.dev0
 - rapids-logger==0.2.*,>=0.0.0a0

conda/environments/all_cuda-130_arch-aarch64.yaml

@@ -31,6 +31,7 @@ dependencies:
 - pre-commit
 - pytest
 - pytest-cov
+- pytest-xdist
 - python>=3.10,<3.14
 - rapids-build-backend>=0.4.0,<0.5.0.dev0
 - rapids-logger==0.2.*,>=0.0.0a0

conda/environments/all_cuda-130_arch-x86_64.yaml

@@ -31,6 +31,7 @@ dependencies:
 - pre-commit
 - pytest
 - pytest-cov
+- pytest-xdist
 - python>=3.10,<3.14
 - rapids-build-backend>=0.4.0,<0.5.0.dev0
 - rapids-logger==0.2.*,>=0.0.0a0

dependencies.yaml

@@ -312,6 +312,7 @@ dependencies:
         packages:
           - pytest
           - pytest-cov
+          - pytest-xdist
     specific:
       - output_types: [conda, requirements, pyproject]
         matrices:

python/rmm/pyproject.toml

@@ -52,6 +52,7 @@ test = [
     "numba>=0.60.0,<0.62.0a0",
     "pytest",
     "pytest-cov",
+    "pytest-xdist",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]