Skip to content

chore: rework virtual environments creation for tests #2356

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Apr 14, 2025
+45 −104
Merged

chore: rework virtual environments creation for tests #2356

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 15 additions & 53 deletions cibuildwheel/platforms/macos.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,25 +241,6 @@ def setup_python(
# we version pip ourselves, so we don't care about pip version checking
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"

# upgrade pip to the version matching our constraints
# if necessary, reinstall it to ensure that it's available on PATH as 'pip'
if build_frontend == "build[uv]":
assert uv_path is not None
pip = [str(uv_path), "pip"]
else:
pip = ["python", "-m", "pip"]

if not use_uv:
call(
*pip,
"install",
"--upgrade",
"pip",
*constraint_flags(dependency_constraint),
env=env,
cwd=venv_path,
)

# Apply our environment after pip is ready
env = environment.as_dictionary(prev_environment=env)

Expand Down Expand Up @@ -434,8 +415,7 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.environment,
build_frontend.name,
)
if not use_uv:
pip_version = get_pip_version(env)
pip_version = None if use_uv else get_pip_version(env)

compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
if compatible_wheel:
Expand All @@ -460,7 +440,7 @@ def build(options: Options, tmp_path: Path) -> None:
)

build_env = env.copy()
if not use_uv:
if pip_version is not None:
build_env["VIRTUALENV_PIP"] = pip_version
if constraints_path:
combine_constraints(
Expand Down Expand Up @@ -614,19 +594,6 @@ def build(options: Options, tmp_path: Path) -> None:
else f"Testing wheel on {testing_arch}..."
)

# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
if not use_uv:
call(
"pip",
"install",
"virtualenv",
*constraint_flags(constraints_path),
env=env,
)

venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}"

arch_prefix = []
uv_arch_args = []
if testing_arch != machine_arch:
Expand All @@ -642,29 +609,24 @@ def build(options: Options, tmp_path: Path) -> None:
call_with_arch = functools.partial(call, *arch_prefix)
shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c")

# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}"
virtualenv_env = virtualenv(
config.version,
base_python,
venv_dir,
None,
use_uv=use_uv,
env=env,
pip_version=pip_version,
)
if use_uv:
pip_install = functools.partial(call, *pip, "install", *uv_arch_args)
call("uv", "venv", venv_dir, f"--python={base_python}", env=env)
else:
pip_install = functools.partial(call_with_arch, *pip, "install")
# Use pip version from the initial env to ensure determinism
venv_args = [
"--no-periodic-update",
f"--pip={pip_version}",
"--no-setuptools",
"--no-wheel",
]
call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env)

virtualenv_env = env.copy()

virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target()
virtualenv_env["PATH"] = os.pathsep.join(
[
str(venv_dir / "bin"),
virtualenv_env["PATH"],
]
)
virtualenv_env["VIRTUAL_ENV"] = str(venv_dir)

# check that we are using the Python from the virtual environment
call_with_arch("which", "python", env=virtualenv_env)
Expand Down
53 changes: 10 additions & 43 deletions cibuildwheel/platforms/windows.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,21 +265,6 @@ def setup_python(
env["PYTHON_ARCH"] = python_configuration.arch
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"

# upgrade pip to the version matching our constraints
# if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe'
if not use_uv:
call(
"python",
"-m",
"pip",
"install",
"--upgrade",
"pip",
*constraint_flags(dependency_constraint),
env=env,
cwd=venv_path,
)

# update env with results from CIBW_ENVIRONMENT
env = environment.as_dictionary(prev_environment=env)

Expand Down Expand Up @@ -378,8 +363,7 @@ def build(options: Options, tmp_path: Path) -> None:
build_options.environment,
build_frontend.name,
)
if not use_uv:
pip_version = get_pip_version(env)
pip_version = None if use_uv else get_pip_version(env)

compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
if compatible_wheel:
Expand Down Expand Up @@ -407,7 +391,7 @@ def build(options: Options, tmp_path: Path) -> None:
)

build_env = env.copy()
if not use_uv:
if pip_version is not None:
build_env["VIRTUALENV_PIP"] = pip_version

if constraints_path:
Expand Down Expand Up @@ -486,33 +470,16 @@ def build(options: Options, tmp_path: Path) -> None:
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
if not use_uv:
call(
"pip", "install", "virtualenv", *constraint_flags(constraints_path), env=env
)

venv_dir = identifier_tmp_dir / "venv-test"

if use_uv:
call("uv", "venv", venv_dir, f"--python={base_python}", env=env)
else:
# Use pip version from the initial env to ensure determinism
venv_args = [
"--no-periodic-update",
f"--pip={pip_version}",
"--no-setuptools",
"--no-wheel",
]
call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env)

virtualenv_env = env.copy()
virtualenv_env["PATH"] = os.pathsep.join(
[
str(venv_dir / "Scripts"),
virtualenv_env["PATH"],
]
virtualenv_env = virtualenv(
config.version,
base_python,
venv_dir,
None,
use_uv=use_uv,
env=env,
pip_version=pip_version,
)
virtualenv_env["VIRTUAL_ENV"] = str(venv_dir)

# check that we are using the Python from the virtual environment
call("where", "python", env=virtualenv_env)
Expand Down
28 changes: 20 additions & 8 deletions cibuildwheel/venv.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ def virtualenv(
dependency_constraint: Path | None,
*,
use_uv: bool,
env: dict[str, str] | None = None,
pip_version: str | None = None,
) -> dict[str, str]:
"""
Create a virtual environment. If `use_uv` is True,
Expand All @@ -109,16 +111,17 @@ def virtualenv(
call("uv", "venv", venv_path, "--python", python)
else:
virtualenv_app = _ensure_virtualenv(version)
pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint)
additional_flags = [f"--pip={pip_constraint}", "--no-setuptools", "--no-wheel"]
if pip_version is None:
pip_version = _parse_pip_constraint_for_virtualenv(dependency_constraint)
additional_flags = [f"--pip={pip_version}", "--no-setuptools", "--no-wheel"]

# Using symlinks to pre-installed seed packages is really the fastest way to get a virtual
# environment. The initial cost is a bit higher but reusing is much faster.
# Windows does not always allow symlinks so just disabling for now.
# Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the
# version of pip that will end-up installed.
# c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder
if not _IS_WIN and pip_constraint != "embed" and Version(pip_constraint) >= Version("19.3"):
if not _IS_WIN and pip_version != "embed" and Version(pip_version) >= Version("19.3"):
additional_flags.append("--symlink-app-data")

call(
Expand All @@ -132,12 +135,21 @@ def virtualenv(
python,
venv_path,
)

paths = [str(venv_path), str(venv_path / "Scripts")] if _IS_WIN else [str(venv_path / "bin")]
env = os.environ.copy()
env["PATH"] = os.pathsep.join([*paths, env["PATH"]])
env["VIRTUAL_ENV"] = str(venv_path)
return env
venv_env = os.environ.copy() if env is None else env.copy()
venv_env["PATH"] = os.pathsep.join([*paths, venv_env["PATH"]])
venv_env["VIRTUAL_ENV"] = str(venv_path)
if not use_uv and pip_version == "embed":
call(
"pip",
"install",
"--upgrade",
"pip",
*constraint_flags(dependency_constraint),
env=venv_env,
cwd=venv_path,
)
return venv_env


def find_uv() -> Path | None:
Expand Down
Loading