From 351548568a514368f213b44f921d2c61badf7841 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 22 Oct 2025 13:48:11 +0800 Subject: [PATCH 01/12] Add test-execution-args option. --- bin/generate_schema.py | 4 ++ cibuildwheel/options.py | 8 ++++ .../resources/cibuildwheel.schema.json | 39 +++++++++++++++++++ cibuildwheel/resources/defaults.toml | 1 + 4 files changed, 52 insertions(+) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index fe5f09187..101f3ecd7 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -224,6 +224,9 @@ test-environment: description: Set environment variables for the test environment type: string_table + test-execution-args: + description: Additional arguments for the test runner to use when configuring the test environment + type: string_array """ schema = yaml.safe_load(starter) @@ -304,6 +307,7 @@ test-sources: {"$ref": "#/$defs/inherit"} test-requires: {"$ref": "#/$defs/inherit"} test-environment: {"$ref": "#/$defs/inherit"} + test-execution-args: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 22dc48d1a..a2a5aeeed 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -110,6 +110,7 @@ class BuildOptions: test_extras: str test_groups: list[str] test_environment: ParsedEnvironment + test_execution_args: list[str] | None build_verbosity: int build_frontend: BuildFrontendConfig config_settings: str @@ -761,6 +762,12 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: msg = f"Malformed environment option {test_environment_config!r}" raise errors.ConfigurationError(msg) from e + test_execution_args = shlex.split( + self.reader.get( + "test-execution-args", option_format=ListFormat(sep=" ", quote=shlex.quote) + ) + ) + test_requires = self.reader.get( "test-requires", option_format=ListFormat(sep=" ") ).split() @@ -868,6 +875,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command=test_command, test_sources=test_sources, test_environment=test_environment, + test_execution_args=test_execution_args, test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, test_groups=test_groups, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index bae1f9eb5..393d54017 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -569,6 +569,21 @@ ], "title": "CIBW_TEST_ENVIRONMENT" }, + "test-execution-args": { + "description": "Additional arguments for the test runner to use when configuring the test environment", + "oneOf": [ + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "title": "CIBW_TEST_EXECUTION_ARGS" + }, "overrides": { "type": "array", "description": "An overrides array", @@ -638,6 +653,9 @@ }, "test-environment": { "$ref": "#/$defs/inherit" + }, + "test-execution-args": { + "$ref": "#/$defs/inherit" } } }, @@ -748,6 +766,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } } @@ -876,6 +897,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } }, @@ -936,6 +960,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } }, @@ -1009,6 +1036,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } }, @@ -1069,6 +1099,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } }, @@ -1129,6 +1162,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } }, @@ -1189,6 +1225,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-execution-args": { + "$ref": "#/properties/test-execution-args" } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index d5c176f27..f94b590a7 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,6 +25,7 @@ test-requires = [] test-extras = [] test-groups = [] test-environment = {} +test-execution-args = [] container-engine = "docker" From 14fb8d11ad18fa2dc6019ab27778311a8ffc299f Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 22 Oct 2025 13:55:55 +0800 Subject: [PATCH 02/12] Add usage of test-execution-args. --- cibuildwheel/platforms/android.py | 15 +++++++++++++-- cibuildwheel/platforms/ios.py | 5 +++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index cb57b907a..f4bf9d74f 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -635,17 +635,28 @@ def test_wheel(state: BuildState, wheel: Path) -> None: ) raise errors.FatalError(msg) + # By default, run on a testbed managed emulator running the newest supported + # Android version. However, if the user specifies a --managed or --connected + # test execution argument, that argument takes precedence. + if state.options.test_execution_args and ( + "--managed" in state.options.test_execution_args + or "--connected" in state.options.test_execution_args + ): + simulator_args = [] + else: + simulator_args = ["--managed", "maxVersion"] + # Run the test app. call( state.python_dir / "android.py", "test", - "--managed", - "maxVersion", "--site-packages", site_packages_dir, "--cwd", cwd_dir, + *simulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), + *(state.options.test_execution_args if state.options.test_execution_args else []), *test_args, env=state.build_env, ) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index eba04c35d..8934ee106 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -658,6 +658,11 @@ def build(options: Options, tmp_path: Path) -> None: testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), + *( + build_options.test_execution_args + if build_options.test_execution_args + else [] + ), "--", *final_command, env=test_env, From 5d3e2abc8cd6aa32926ab81b3aff7002436346fb Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 22 Oct 2025 13:58:09 +0800 Subject: [PATCH 03/12] Add CI configuration to use test-execution-args. --- .github/workflows/test.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52621caf9..f84e19c7f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,9 +79,16 @@ jobs: - os: macos-15 python_version: '3.13' test_select: ios + # Github Actions macOS-15 runner has disk performance issues; so it + # has problems using any simulator that isn't pre-warmed. + # Unfortunately, the simulator that it picks by default isn't + # pre-warmed. So - we need to explicitly select a simulator. + test_execution_args: '--simulator "iPhone 16e,OS=18.5"' - os: macos-15-intel python_version: '3.13' test_select: android + # Exercise Android on a non-default simulator + test_execution_args: '--managed minVersion' - os: macos-15 python_version: '3.13' test_select: android @@ -154,6 +161,7 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} - name: Run a sample build (GitHub Action, only) uses: ./ @@ -179,6 +187,7 @@ jobs: uses: ./ env: CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -202,6 +211,8 @@ jobs: path: wheelhouse/*.whl - name: Test cibuildwheel + env: + CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} run: | uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} From 8240dfe67ad96659b9b9b375b563e9e07b8adc83 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 22 Oct 2025 14:18:25 +0800 Subject: [PATCH 04/12] Document the test-execution-args setting. --- README.md | 8 ++++---- docs/options.md | 35 +++++++++++++++++++++++++++++++++++ docs/platforms.md | 6 ++++++ 3 files changed, 45 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5266ee697..1f6dd369d 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ Usage | | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | |-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³⁵ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³⁵ | +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | | Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | | CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | | Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | @@ -70,7 +70,6 @@ Usage ² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).
-⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](platforms/#ios-system-requirements)
when building iOS wheels. @@ -160,12 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | | | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | | | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | +| | [`test-execution-args`](https://cibuildwheel.pypa.io/en/stable/options/#test-execution-args) | Define additional arguments that will be passed to the command that runs the tests. | | **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | | | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). diff --git a/docs/options.md b/docs/options.md index 1c4e89d89..6a9c1f8bb 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1669,6 +1669,41 @@ Platform-specific environment variables are also available:
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` +### `test-execution-args` {: #test-execution-args toml env-var } + +> Define additional arguments that will be passed to the command that runs the tests. + +A list of arguments that will be used by the test runner when running tests. This is used by environments where the execution environment can be customized. For example, mobile platforms will use these arguments to control the execution of the testbed application that is used to run tests. + +This option will be ignored on platforms that do not have a separate test runner. + +Platform-specific environment variables are also available:
+`CIBW_TEST_EXECUTION_ARGS_MACOS` | `CIBW_TEST_EXECUTION_ARGS_WINDOWS` | `CIBW_TEST_EXECUTION_ARGS_LINUX` | `CIBW_TEST_EXECUTION_ARGS_ANDROID` |`CIBW_TEST_EXECUTION_ARGS_IOS` | `CIBW_TEST_EXECUTION_ARGS_PYODIDE` + +#### Examples + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.ios] + # Run the tests on an iPhone 16e simulator running iOS 18.5. + test-execution-args = ["--simulator='iPhone 16e,OS=18.5'"] + + [tool.cibuildwheel.android] + # Run the Android tests on the minimum supported Android version. + test-execution-args = ["--managed", "minVersion"] + ``` + +!!! tab examples "Environment variables" + + ```yaml + # Run the tests on an iPhone 16e simulator running iOS 18.5. + CIBW_EXECUTION_ARGS_IOS: --simulator='iPhone 16e,OS=18.5' + + # Run the Android tests on the minimum supported Android version. + CIBW_EXECUTION_ARGS_ANDROID: --managed minVersion + ``` + ## Debugging diff --git a/docs/platforms.md b/docs/platforms.md index 1cb931ebc..ce2f7b493 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,6 +233,8 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. +Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The most common arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. + Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -321,3 +323,7 @@ If tests have been configured, the test suite will be executed on the simulator The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. +Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The most common argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. For example, `test_execution_args = ["--simulator", "iPhone 16e,OS=18.5"]` would specify the use of an iPhone 16e simulator running OS 18.5. + +!!! note + The `macos-15` image on GitHub Actions and Azure has a [known performance issue](https://github.com/actions/runner-images/issues/12777) that can lead to iOS simulators failing to start if they have not been pre-warmed. At this time, the workaround is to explicitly specify a simulator that is pre-warmed by default on the image; the iPhone 16e simulator running iOS 18.5 is one such image. This workaround is only needed on the `macos-15`; the `macos-14` image is not affected. From d238fb9443495f3f5cb46c826ad44d79b1d5b733 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 23 Oct 2025 08:18:56 +0800 Subject: [PATCH 05/12] Simplify code using or syntax instead of inline if. Co-authored-by: Malcolm Smith --- cibuildwheel/platforms/android.py | 2 +- cibuildwheel/platforms/ios.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index f4bf9d74f..63b7fe59c 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -656,7 +656,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: cwd_dir, *simulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), - *(state.options.test_execution_args if state.options.test_execution_args else []), + *(state.options.test_execution_args or []), *test_args, env=state.build_env, ) diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 8934ee106..4f253a476 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -658,11 +658,7 @@ def build(options: Options, tmp_path: Path) -> None: testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), - *( - build_options.test_execution_args - if build_options.test_execution_args - else [] - ), + *(build_options.test_execution_args or []), "--", *final_command, env=test_env, From 080b33a72bda7037133fc74ecbd279488bc78543 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 23 Oct 2025 08:34:11 +0800 Subject: [PATCH 06/12] Clarified some Android-specific terminology, and added details about the default args to the test runner. --- cibuildwheel/platforms/android.py | 6 +++--- docs/platforms.md | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 63b7fe59c..2c455f098 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -642,9 +642,9 @@ def test_wheel(state: BuildState, wheel: Path) -> None: "--managed" in state.options.test_execution_args or "--connected" in state.options.test_execution_args ): - simulator_args = [] + emulator_args = [] else: - simulator_args = ["--managed", "maxVersion"] + emulator_args = ["--managed", "maxVersion"] # Run the test app. call( @@ -654,7 +654,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: site_packages_dir, "--cwd", cwd_dir, - *simulator_args, + *emulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), *(state.options.test_execution_args or []), *test_args, diff --git a/docs/platforms.md b/docs/platforms.md index ce2f7b493..42b08f8a4 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,7 +233,7 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. -Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The most common arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. +Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -323,7 +323,8 @@ If tests have been configured, the test suite will be executed on the simulator The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. -Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The most common argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. For example, `test_execution_args = ["--simulator", "iPhone 16e,OS=18.5"]` would specify the use of an iPhone 16e simulator running OS 18.5. + +Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build_verbosity`](options.md#build_verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. For example, `test_execution_args = ["--simulator", "iPhone 16e,OS=18.5"]` would specify the use of an iPhone 16e simulator running OS 18.5. !!! note The `macos-15` image on GitHub Actions and Azure has a [known performance issue](https://github.com/actions/runner-images/issues/12777) that can lead to iOS simulators failing to start if they have not been pre-warmed. At this time, the workaround is to explicitly specify a simulator that is pre-warmed by default on the image; the iPhone 16e simulator running iOS 18.5 is one such image. This workaround is only needed on the `macos-15`; the `macos-14` image is not affected. From 5c0940c259b8e2c3b4c3dbdc543793ee9b1e6f17 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 28 Oct 2025 16:10:43 +0800 Subject: [PATCH 07/12] Switch to a dict-based test-execution configuration --- .github/workflows/test.yml | 15 +++--- README.md | 4 +- bin/generate_schema.py | 19 +++++-- cibuildwheel/options.py | 36 ++++++++++--- cibuildwheel/platforms/android.py | 9 ++-- cibuildwheel/platforms/ios.py | 26 ++++++++- .../resources/cibuildwheel.schema.json | 54 +++++++++++-------- cibuildwheel/resources/defaults.toml | 2 +- docs/options.md | 18 +++---- docs/platforms.md | 9 ++-- 10 files changed, 126 insertions(+), 66 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f84e19c7f..8d92ae343 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,16 +79,13 @@ jobs: - os: macos-15 python_version: '3.13' test_select: ios - # Github Actions macOS-15 runner has disk performance issues; so it - # has problems using any simulator that isn't pre-warmed. - # Unfortunately, the simulator that it picks by default isn't - # pre-warmed. So - we need to explicitly select a simulator. - test_execution_args: '--simulator "iPhone 16e,OS=18.5"' + # Exercise iOS on a non-default simulator. + test_execution: 'args: --simulator "iPhone 16e,OS=18.5"' - os: macos-15-intel python_version: '3.13' test_select: android # Exercise Android on a non-default simulator - test_execution_args: '--managed minVersion' + test_execution: 'args: --managed minVersion' - os: macos-15 python_version: '3.13' test_select: android @@ -161,7 +158,7 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} CIBW_PLATFORM: ${{ matrix.test_select }} - CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} + CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} - name: Run a sample build (GitHub Action, only) uses: ./ @@ -187,7 +184,7 @@ jobs: uses: ./ env: CIBW_PLATFORM: ${{ matrix.test_select }} - CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} + CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -212,7 +209,7 @@ jobs: - name: Test cibuildwheel env: - CIBW_TEST_EXECUTION_ARGS: ${{ matrix.test_execution_args }} + CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} run: | uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} diff --git a/README.md b/README.md index 1f6dd369d..fb6b79d9c 100644 --- a/README.md +++ b/README.md @@ -159,13 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | | | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | | | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | -| | [`test-execution-args`](https://cibuildwheel.pypa.io/en/stable/options/#test-execution-args) | Define additional arguments that will be passed to the command that runs the tests. | +| | [`test-execution`](https://cibuildwheel.pypa.io/en/stable/options/#test-execution) | Controls how the tests will be executed. | | **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | | | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 101f3ecd7..0592302d0 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -224,9 +224,20 @@ test-environment: description: Set environment variables for the test environment type: string_table - test-execution-args: - description: Additional arguments for the test runner to use when configuring the test environment - type: string_array + test-execution: + description: Additional configuration for the test runner + oneOf: + - type: string + pattern: 'args:' + - type: object + additionalProperties: false + required: [args] + properties: + args: + type: array + items: + type: string + """ schema = yaml.safe_load(starter) @@ -307,7 +318,7 @@ test-sources: {"$ref": "#/$defs/inherit"} test-requires: {"$ref": "#/$defs/inherit"} test-environment: {"$ref": "#/$defs/inherit"} - test-execution-args: {"$ref": "#/$defs/inherit"} + test-execution: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index a2a5aeeed..680b8c636 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -24,7 +24,7 @@ from .selector import BuildSelector, EnableGroup, TestSelector, selector_matches from .typing import PLATFORMS, PlatformName from .util import resources -from .util.helpers import format_safe, strtobool, unwrap +from .util.helpers import format_safe, parse_key_value_string, strtobool, unwrap from .util.packaging import DependencyConstraints MANYLINUX_ARCHS: Final[tuple[str, ...]] = ( @@ -92,6 +92,20 @@ class GlobalOptions: allow_empty: bool +@dataclasses.dataclass(frozen=True) +class TestExecutionConfig: + args: Sequence[str] = () + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string(config_string, ["args"]) + args = config_dict.get("args") or [] + return cls(args=args) + + def options_summary(self) -> str | dict[str, str]: + return {"args": repr(self.args)} + + @dataclasses.dataclass(frozen=True, kw_only=True) class BuildOptions: globals: GlobalOptions @@ -110,7 +124,7 @@ class BuildOptions: test_extras: str test_groups: list[str] test_environment: ParsedEnvironment - test_execution_args: list[str] | None + test_execution: TestExecutionConfig build_verbosity: int build_frontend: BuildFrontendConfig config_settings: str @@ -762,11 +776,19 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: msg = f"Malformed environment option {test_environment_config!r}" raise errors.ConfigurationError(msg) from e - test_execution_args = shlex.split( - self.reader.get( - "test-execution-args", option_format=ListFormat(sep=" ", quote=shlex.quote) - ) + test_execution_str = self.reader.get( + "test-execution", + env_plat=False, + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ) + if not test_execution_str or test_execution_str == "default": + test_execution = TestExecutionConfig() + else: + try: + test_execution = TestExecutionConfig.from_config_string(test_execution_str) + except ValueError as e: + msg = f"Failed to parse test execution config. {e}" + raise errors.ConfigurationError(msg) from e test_requires = self.reader.get( "test-requires", option_format=ListFormat(sep=" ") @@ -875,7 +897,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command=test_command, test_sources=test_sources, test_environment=test_environment, - test_execution_args=test_execution_args, + test_execution=test_execution, test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, test_groups=test_groups, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 6909d105b..4bc4a83b2 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -641,10 +641,9 @@ def test_wheel(state: BuildState, wheel: Path) -> None: # By default, run on a testbed managed emulator running the newest supported # Android version. However, if the user specifies a --managed or --connected # test execution argument, that argument takes precedence. - if state.options.test_execution_args and ( - "--managed" in state.options.test_execution_args - or "--connected" in state.options.test_execution_args - ): + test_execution_args = state.options.test_execution.args + + if any(arg.startswith(("--managed", "--connected")) for arg in test_execution_args): emulator_args = [] else: emulator_args = ["--managed", "maxVersion"] @@ -659,7 +658,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: cwd_dir, *emulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), - *(state.options.test_execution_args or []), + *test_execution_args, "--", *test_args, env=state.build_env, diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 4f253a476..58b61f5b9 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -2,6 +2,7 @@ import dataclasses import os +import platform import shlex import shutil import subprocess @@ -653,12 +654,35 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.FatalError(msg) + test_execution_args = build_options.test_execution.args + + # 2025-10: The GitHub Actions macos-15 runner has a known issue where + # the default simulator won't start due to a disk performance issue; + # see https://github.com/actions/runner-images/issues/12777 for details. + # In the meantime, if it looks like we're running on a GitHub Actions + # macos-15 runner, use a simulator that is known to work, unless the + # user explicitly specifies a simulator. + os_version, _, arch = platform.mac_ver() + if ( + "GITHUB_ACTIONS" in os.environ + and os_version.startswith("15.") + and arch == "arm64" + and not any( + arg.startswith("--simulator") for arg in test_execution_args + ) + ): + test_execution_args = [ + "--simulator", + "iPhone 16e,OS=18.5", + *test_execution_args, + ] + call( "python", testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), - *(build_options.test_execution_args or []), + *test_execution_args, "--", *final_command, env=test_env, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 393d54017..dcf53282a 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -569,20 +569,30 @@ ], "title": "CIBW_TEST_ENVIRONMENT" }, - "test-execution-args": { - "description": "Additional arguments for the test runner to use when configuring the test environment", + "test-execution": { + "description": "Additional configuration for the test runner", "oneOf": [ { - "type": "string" + "type": "string", + "pattern": "args:" }, { - "type": "array", - "items": { - "type": "string" + "type": "object", + "additionalProperties": false, + "required": [ + "args" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + } } } ], - "title": "CIBW_TEST_EXECUTION_ARGS" + "title": "CIBW_TEST_EXECUTION" }, "overrides": { "type": "array", @@ -654,7 +664,7 @@ "test-environment": { "$ref": "#/$defs/inherit" }, - "test-execution-args": { + "test-execution": { "$ref": "#/$defs/inherit" } } @@ -767,8 +777,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } } @@ -898,8 +908,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } }, @@ -961,8 +971,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } }, @@ -1037,8 +1047,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } }, @@ -1100,8 +1110,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } }, @@ -1163,8 +1173,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } }, @@ -1226,8 +1236,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution-args": { - "$ref": "#/properties/test-execution-args" + "test-execution": { + "$ref": "#/properties/test-execution" } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index f94b590a7..73be3c7af 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,7 +25,7 @@ test-requires = [] test-extras = [] test-groups = [] test-environment = {} -test-execution-args = [] +test-execution = {} container-engine = "docker" diff --git a/docs/options.md b/docs/options.md index b2acf9408..08cd2a162 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1672,16 +1672,16 @@ Platform-specific environment variables are also available:
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` -### `test-execution-args` {: #test-execution-args toml env-var } +### `test-execution` {: #test-execution toml env-var } -> Define additional arguments that will be passed to the command that runs the tests. +> Controls how the tests will be executed. -A list of arguments that will be used by the test runner when running tests. This is used by environments where the execution environment can be customized. For example, mobile platforms will use these arguments to control the execution of the testbed application that is used to run tests. +In simple desktop environments, the test suite will be executed using the command defined by [`test-command`][test-command]. However, on some platforms, the test is executed using a test runner; the `test-execution` setting is used to control that test execution environment. The value of the setting is a dictionary that can contain additional keys, depending on the platform being targeted. -This option will be ignored on platforms that do not have a separate test runner. +On mobile platforms (iOS and Android), a testbed project is used to run the tests. The `test-execution` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. Platform-specific environment variables are also available:
-`CIBW_TEST_EXECUTION_ARGS_MACOS` | `CIBW_TEST_EXECUTION_ARGS_WINDOWS` | `CIBW_TEST_EXECUTION_ARGS_LINUX` | `CIBW_TEST_EXECUTION_ARGS_ANDROID` |`CIBW_TEST_EXECUTION_ARGS_IOS` | `CIBW_TEST_EXECUTION_ARGS_PYODIDE` +`CIBW_TEST_EXECUTION_ANDROID` |`CIBW_TEST_EXECUTION_IOS` #### Examples @@ -1690,21 +1690,21 @@ Platform-specific environment variables are also available:
```toml [tool.cibuildwheel.ios] # Run the tests on an iPhone 16e simulator running iOS 18.5. - test-execution-args = ["--simulator='iPhone 16e,OS=18.5'"] + test-execution = { args = ["--simulator='iPhone 16e,OS=18.5'"] } [tool.cibuildwheel.android] # Run the Android tests on the minimum supported Android version. - test-execution-args = ["--managed", "minVersion"] + test-execution = { args = ["--managed", "minVersion"] } ``` !!! tab examples "Environment variables" ```yaml # Run the tests on an iPhone 16e simulator running iOS 18.5. - CIBW_EXECUTION_ARGS_IOS: --simulator='iPhone 16e,OS=18.5' + CIBW_EXECUTION_IOS: "args: --simulator='iPhone 16e,OS=18.5'" # Run the Android tests on the minimum supported Android version. - CIBW_EXECUTION_ARGS_ANDROID: --managed minVersion + CIBW_EXECUTION_ANDROID: "args: --managed minVersion" ``` diff --git a/docs/platforms.md b/docs/platforms.md index 42b08f8a4..13353d511 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,7 +233,7 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. -Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. +Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -322,9 +322,6 @@ If tests have been configured, the test suite will be executed on the simulator The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. -The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. +The test process uses the [same testbed used by CPython itself](https://github.com/python/cpython/tree/main/Apple/iOS#testing-python-on-ios) to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. -Any arguments specified using [`test-execution-args`](options.md#test-execution-args) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build_verbosity`](options.md#build_verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. For example, `test_execution_args = ["--simulator", "iPhone 16e,OS=18.5"]` would specify the use of an iPhone 16e simulator running OS 18.5. - -!!! note - The `macos-15` image on GitHub Actions and Azure has a [known performance issue](https://github.com/actions/runner-images/issues/12777) that can lead to iOS simulators failing to start if they have not been pre-warmed. At this time, the workaround is to explicitly specify a simulator that is pre-warmed by default on the image; the iPhone 16e simulator running iOS 18.5 is one such image. This workaround is only needed on the `macos-15`; the `macos-14` image is not affected. +Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build_verbosity`](options.md#build_verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available. From e70637099969ef2b73b7f278ec81d4c03998bf11 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 28 Oct 2025 17:54:56 +0800 Subject: [PATCH 08/12] Add tests for test-execution parsing. --- bin/generate_schema.py | 4 +++ cibuildwheel/options.py | 4 +-- .../resources/cibuildwheel.schema.json | 8 +++++ unit_test/options_test.py | 33 +++++++++++++++++++ 4 files changed, 47 insertions(+), 2 deletions(-) diff --git a/bin/generate_schema.py b/bin/generate_schema.py index 0592302d0..feef6cbcd 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -227,6 +227,10 @@ test-execution: description: Additional configuration for the test runner oneOf: + - type: string + pattern: '^$' + - type: object + additionalProperties: false - type: string pattern: 'args:' - type: object diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 680b8c636..163d0e361 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -98,7 +98,7 @@ class TestExecutionConfig: @classmethod def from_config_string(cls, config_string: str) -> Self: - config_dict = parse_key_value_string(config_string, ["args"]) + config_dict = parse_key_value_string(config_string, [], ["args"]) args = config_dict.get("args") or [] return cls(args=args) @@ -781,7 +781,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: env_plat=False, option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ) - if not test_execution_str or test_execution_str == "default": + if not test_execution_str: test_execution = TestExecutionConfig() else: try: diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index dcf53282a..6dc7b6e9a 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -576,6 +576,14 @@ "type": "string", "pattern": "args:" }, + { + "type": "string", + "pattern": "^$" + }, + { + "type": "object", + "additionalProperties": false + }, { "type": "object", "additionalProperties": false, diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 52f2f1adf..bdedbcb39 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -2,6 +2,7 @@ import platform as platform_module import textwrap import unittest.mock +from collections.abc import Sequence from pathlib import Path from typing import Literal @@ -626,6 +627,38 @@ def test_get_build_frontend_extra_flags_warning( mock_warning.assert_called_once() +@pytest.mark.parametrize( + ("definition", "expected"), + [ + ("", ()), + ("test-execution = {}", ()), + ('test-execution = {args = ""}', []), + ('test-execution = "args: --simulator foo"', ["--simulator", "foo"]), + ('test-execution = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]), + ], +) +def test_test_execution_handling( + tmp_path: Path, definition: str, expected_args: Sequence[str] | None +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {definition} + """ + ) + ) + + options = Options(platform="ios", command_line_arguments=args, env={}) + + local = options.build_options("cp313-ios_13_0_arm64_iphoneos") + assert local.test_execution.args == expected_args + + @pytest.mark.parametrize( ("definition", "expected"), [ From c33f4fe9e368775dbc5d1e21dc2f7fda21edd3cd Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Tue, 28 Oct 2025 17:57:06 +0800 Subject: [PATCH 09/12] Add all the files before pushing... --- unit_test/options_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_test/options_test.py b/unit_test/options_test.py index bdedbcb39..c1f3edd5d 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -628,7 +628,7 @@ def test_get_build_frontend_extra_flags_warning( @pytest.mark.parametrize( - ("definition", "expected"), + ("definition", "expected_args"), [ ("", ()), ("test-execution = {}", ()), From 9b24b8022e550fdf36e992f146c2333375e77c54 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 10:09:40 +0800 Subject: [PATCH 10/12] Add note about default Android version for testbed. --- docs/platforms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/platforms.md b/docs/platforms.md index 13353d511..aca239824 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,7 +233,7 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. -Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. +Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`. Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: From 40514002d10bd1bc96dfb644710bba1ab9502ec1 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 30 Oct 2025 06:39:02 +0800 Subject: [PATCH 11/12] Improve description of test-execution setting. Co-authored-by: Joe Rickerby --- docs/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/options.md b/docs/options.md index 08cd2a162..5d3008550 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1676,9 +1676,9 @@ Platform-specific environment variables are also available:
> Controls how the tests will be executed. -In simple desktop environments, the test suite will be executed using the command defined by [`test-command`][test-command]. However, on some platforms, the test is executed using a test runner; the `test-execution` setting is used to control that test execution environment. The value of the setting is a dictionary that can contain additional keys, depending on the platform being targeted. +On desktop environments, the tests are executed on the same machine/container as the wheel was built. However on Android and iOS, the tests are run inside a virtual machine – a simulator or emulator – representing the target. -On mobile platforms (iOS and Android), a testbed project is used to run the tests. The `test-execution` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. +For these embedded platforms, a testbed project is used to run the tests. The `test-execution` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. Platform-specific environment variables are also available:
`CIBW_TEST_EXECUTION_ANDROID` |`CIBW_TEST_EXECUTION_IOS` From dcd0743844bd0392dcfd3e236eebe1324090687e Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Fri, 31 Oct 2025 13:39:24 +0800 Subject: [PATCH 12/12] Switch to using test-runtime. --- .github/workflows/test.yml | 10 ++--- README.md | 4 +- bin/generate_schema.py | 4 +- cibuildwheel/options.py | 18 ++++---- cibuildwheel/platforms/android.py | 6 +-- cibuildwheel/platforms/ios.py | 10 ++--- .../resources/cibuildwheel.schema.json | 42 +++++++++---------- cibuildwheel/resources/defaults.toml | 2 +- docs/options.md | 14 +++---- docs/platforms.md | 4 +- unit_test/options_test.py | 13 +++--- 11 files changed, 64 insertions(+), 63 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d92ae343..43fe9cd6b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -80,12 +80,12 @@ jobs: python_version: '3.13' test_select: ios # Exercise iOS on a non-default simulator. - test_execution: 'args: --simulator "iPhone 16e,OS=18.5"' + test_runtime: 'args: --simulator "iPhone 16e,OS=18.5"' - os: macos-15-intel python_version: '3.13' test_select: android # Exercise Android on a non-default simulator - test_execution: 'args: --managed minVersion' + test_runtime: 'args: --managed minVersion' - os: macos-15 python_version: '3.13' test_select: android @@ -158,7 +158,7 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} CIBW_PLATFORM: ${{ matrix.test_select }} - CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} - name: Run a sample build (GitHub Action, only) uses: ./ @@ -184,7 +184,7 @@ jobs: uses: ./ env: CIBW_PLATFORM: ${{ matrix.test_select }} - CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -209,7 +209,7 @@ jobs: - name: Test cibuildwheel env: - CIBW_TEST_EXECUTION: ${{ matrix.test_execution }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} run: | uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} diff --git a/README.md b/README.md index fb6b79d9c..f3847b88e 100644 --- a/README.md +++ b/README.md @@ -159,13 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | | | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | | | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | -| | [`test-execution`](https://cibuildwheel.pypa.io/en/stable/options/#test-execution) | Controls how the tests will be executed. | +| | [`test-runtime`](https://cibuildwheel.pypa.io/en/stable/options/#test-runtime) | Controls how the tests will be executed. | | **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | | | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). diff --git a/bin/generate_schema.py b/bin/generate_schema.py index feef6cbcd..611d19e5a 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -224,7 +224,7 @@ test-environment: description: Set environment variables for the test environment type: string_table - test-execution: + test-runtime: description: Additional configuration for the test runner oneOf: - type: string @@ -322,7 +322,7 @@ test-sources: {"$ref": "#/$defs/inherit"} test-requires: {"$ref": "#/$defs/inherit"} test-environment: {"$ref": "#/$defs/inherit"} - test-execution: {"$ref": "#/$defs/inherit"} + test-runtime: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 163d0e361..af946a883 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -93,7 +93,7 @@ class GlobalOptions: @dataclasses.dataclass(frozen=True) -class TestExecutionConfig: +class TestRuntimeConfig: args: Sequence[str] = () @classmethod @@ -124,7 +124,7 @@ class BuildOptions: test_extras: str test_groups: list[str] test_environment: ParsedEnvironment - test_execution: TestExecutionConfig + test_runtime: TestRuntimeConfig build_verbosity: int build_frontend: BuildFrontendConfig config_settings: str @@ -776,18 +776,18 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: msg = f"Malformed environment option {test_environment_config!r}" raise errors.ConfigurationError(msg) from e - test_execution_str = self.reader.get( - "test-execution", + test_runtime_str = self.reader.get( + "test-runtime", env_plat=False, option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), ) - if not test_execution_str: - test_execution = TestExecutionConfig() + if not test_runtime_str: + test_runtime = TestRuntimeConfig() else: try: - test_execution = TestExecutionConfig.from_config_string(test_execution_str) + test_runtime = TestRuntimeConfig.from_config_string(test_runtime_str) except ValueError as e: - msg = f"Failed to parse test execution config. {e}" + msg = f"Failed to parse test runtime config. {e}" raise errors.ConfigurationError(msg) from e test_requires = self.reader.get( @@ -897,7 +897,7 @@ def _compute_build_options(self, identifier: str | None) -> BuildOptions: test_command=test_command, test_sources=test_sources, test_environment=test_environment, - test_execution=test_execution, + test_runtime=test_runtime, test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, test_groups=test_groups, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index 4bc4a83b2..13438250e 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -641,9 +641,9 @@ def test_wheel(state: BuildState, wheel: Path) -> None: # By default, run on a testbed managed emulator running the newest supported # Android version. However, if the user specifies a --managed or --connected # test execution argument, that argument takes precedence. - test_execution_args = state.options.test_execution.args + test_runtime_args = state.options.test_runtime.args - if any(arg.startswith(("--managed", "--connected")) for arg in test_execution_args): + if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): emulator_args = [] else: emulator_args = ["--managed", "maxVersion"] @@ -658,7 +658,7 @@ def test_wheel(state: BuildState, wheel: Path) -> None: cwd_dir, *emulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), - *test_execution_args, + *test_runtime_args, "--", *test_args, env=state.build_env, diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index 58b61f5b9..cf9ba4eba 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -654,7 +654,7 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.FatalError(msg) - test_execution_args = build_options.test_execution.args + test_runtime_args = build_options.test_runtime.args # 2025-10: The GitHub Actions macos-15 runner has a known issue where # the default simulator won't start due to a disk performance issue; @@ -668,13 +668,13 @@ def build(options: Options, tmp_path: Path) -> None: and os_version.startswith("15.") and arch == "arm64" and not any( - arg.startswith("--simulator") for arg in test_execution_args + arg.startswith("--simulator") for arg in test_runtime_args ) ): - test_execution_args = [ + test_runtime_args = [ "--simulator", "iPhone 16e,OS=18.5", - *test_execution_args, + *test_runtime_args, ] call( @@ -682,7 +682,7 @@ def build(options: Options, tmp_path: Path) -> None: testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), - *test_execution_args, + *test_runtime_args, "--", *final_command, env=test_env, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 6dc7b6e9a..2c52aada4 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -569,13 +569,9 @@ ], "title": "CIBW_TEST_ENVIRONMENT" }, - "test-execution": { + "test-runtime": { "description": "Additional configuration for the test runner", "oneOf": [ - { - "type": "string", - "pattern": "args:" - }, { "type": "string", "pattern": "^$" @@ -584,6 +580,10 @@ "type": "object", "additionalProperties": false }, + { + "type": "string", + "pattern": "args:" + }, { "type": "object", "additionalProperties": false, @@ -600,7 +600,7 @@ } } ], - "title": "CIBW_TEST_EXECUTION" + "title": "CIBW_TEST_RUNTIME" }, "overrides": { "type": "array", @@ -672,7 +672,7 @@ "test-environment": { "$ref": "#/$defs/inherit" }, - "test-execution": { + "test-runtime": { "$ref": "#/$defs/inherit" } } @@ -785,8 +785,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } @@ -916,8 +916,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -979,8 +979,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1055,8 +1055,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1118,8 +1118,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1181,8 +1181,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1244,8 +1244,8 @@ "test-environment": { "$ref": "#/properties/test-environment" }, - "test-execution": { - "$ref": "#/properties/test-execution" + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 73be3c7af..78895bf9a 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,7 +25,7 @@ test-requires = [] test-extras = [] test-groups = [] test-environment = {} -test-execution = {} +test-runtime = {} container-engine = "docker" diff --git a/docs/options.md b/docs/options.md index 5d3008550..88a377d30 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1672,16 +1672,16 @@ Platform-specific environment variables are also available:
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` -### `test-execution` {: #test-execution toml env-var } +### `test-runtime` {: #test-runtime toml env-var } > Controls how the tests will be executed. On desktop environments, the tests are executed on the same machine/container as the wheel was built. However on Android and iOS, the tests are run inside a virtual machine – a simulator or emulator – representing the target. -For these embedded platforms, a testbed project is used to run the tests. The `test-execution` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. +For these embedded platforms, a testbed project is used to run the tests. The `test-runtime` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. Platform-specific environment variables are also available:
-`CIBW_TEST_EXECUTION_ANDROID` |`CIBW_TEST_EXECUTION_IOS` +`CIBW_TEST_RUNTIME_ANDROID` |`CIBW_TEST_RUNTIME_IOS` #### Examples @@ -1690,21 +1690,21 @@ Platform-specific environment variables are also available:
```toml [tool.cibuildwheel.ios] # Run the tests on an iPhone 16e simulator running iOS 18.5. - test-execution = { args = ["--simulator='iPhone 16e,OS=18.5'"] } + test-runtime = { args = ["--simulator='iPhone 16e,OS=18.5'"] } [tool.cibuildwheel.android] # Run the Android tests on the minimum supported Android version. - test-execution = { args = ["--managed", "minVersion"] } + test-runtime = { args = ["--managed", "minVersion"] } ``` !!! tab examples "Environment variables" ```yaml # Run the tests on an iPhone 16e simulator running iOS 18.5. - CIBW_EXECUTION_IOS: "args: --simulator='iPhone 16e,OS=18.5'" + CIBW_TEST_RUNTIME_IOS: "args: --simulator='iPhone 16e,OS=18.5'" # Run the Android tests on the minimum supported Android version. - CIBW_EXECUTION_ANDROID: "args: --managed minVersion" + CIBW_TEST_RUNTIME_ANDROID: "args: --managed minVersion" ``` diff --git a/docs/platforms.md b/docs/platforms.md index aca239824..dee822490 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,7 +233,7 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. -Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build_verbosity`](options.md#build_verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`. +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build-verbosity`](options.md#build-verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`. Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -324,4 +324,4 @@ The iOS test environment can't support running shell scripts, so the [`test-comm The test process uses the [same testbed used by CPython itself](https://github.com/python/cpython/tree/main/Apple/iOS#testing-python-on-ios) to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. -Any arguments specified using [`test-execution`](options.md#test-execution) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build_verbosity`](options.md#build_verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available. +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build-verbosity`](options.md#build-verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available. diff --git a/unit_test/options_test.py b/unit_test/options_test.py index c1f3edd5d..b9847ed5c 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -631,13 +631,14 @@ def test_get_build_frontend_extra_flags_warning( ("definition", "expected_args"), [ ("", ()), - ("test-execution = {}", ()), - ('test-execution = {args = ""}', []), - ('test-execution = "args: --simulator foo"', ["--simulator", "foo"]), - ('test-execution = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]), + ('test-runtime = ""', ()), + ("test-runtime = {}", ()), + ('test-runtime = {args = ""}', []), + ('test-runtime = "args: --simulator foo"', ["--simulator", "foo"]), + ('test-runtime = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]), ], ) -def test_test_execution_handling( +def test_test_runtime_handling( tmp_path: Path, definition: str, expected_args: Sequence[str] | None ) -> None: args = CommandLineArguments.defaults() @@ -656,7 +657,7 @@ def test_test_execution_handling( options = Options(platform="ios", command_line_arguments=args, env={}) local = options.build_options("cp313-ios_13_0_arm64_iphoneos") - assert local.test_execution.args == expected_args + assert local.test_runtime.args == expected_args @pytest.mark.parametrize(