Merge pull request #1347 from davidhewitt/embedding

auto-initialize: new feature to control initializing Python

Author: davidhewitt

.github/workflows/ci.yml

@@ -85,27 +85,28 @@ jobs:
         run: echo LD_LIBRARY_PATH=${pythonLocation}/lib >> $GITHUB_ENV
 
       - name: Build docs
-        run: cargo doc --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
+        run: cargo doc --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
 
-      - name: Build without default features
+      - name: Build (no features)
         run: cargo build --no-default-features --verbose --target ${{ matrix.platform.rust-target }}
 
-      - name: Build with default features
-        run: cargo build --features "num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
+      - name: Build (all additive features)
+        run: cargo build --no-default-features --features "macros num-bigint num-complex hashbrown" --verbose --target ${{ matrix.platform.rust-target }}
 
       # Run tests (except on PyPy, because no embedding API).
       - if: matrix.python-version != 'pypy-3.6'
         name: Test
-        run: cargo test --features "num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
+
       # Run tests again, but in abi3 mode
       - if: matrix.python-version != 'pypy-3.6'
         name: Test (abi3)
-        run: cargo test --no-default-features --features "abi3,macros" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "abi3 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
 
       # Run tests again, for abi3-py36 (the minimal Python version)
       - if: (matrix.python-version != 'pypy-3.6') && (matrix.python-version != '3.6')
         name: Test (abi3-py36)
-        run: cargo test --no-default-features --features "abi3-py36,macros" --target ${{ matrix.platform.rust-target }}
+        run: cargo test --no-default-features --features "abi3-py36 macros num-bigint num-complex hashbrown" --target ${{ matrix.platform.rust-target }}
 
       - name: Test proc-macro code
         run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml --target ${{ matrix.platform.rust-target }}
@@ -125,6 +126,9 @@ jobs:
     env:
       RUST_BACKTRACE: 1
       RUSTFLAGS: "-D warnings"
+      # TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
+      # Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
+      PYO3_CI: 1
 
   coverage:
     needs: [fmt]

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Add support for `#[pyclass(dict)]` and `#[pyclass(weakref)]` with the `abi3` feature on Python 3.9 and up. [#1342](https://github.com/PyO3/pyo3/pull/1342)
 - Add FFI definitions `PyOS_BeforeFork`, `PyOS_AfterFork_Parent`, `PyOS_AfterFork_Child` for Python 3.7 and up. [#1348](https://github.com/PyO3/pyo3/pull/1348)
+- Add `auto-initialize` feature to control whether PyO3 should automatically initialize an embedded Python interpreter. For compatibility this feature is enabled by default in PyO3 0.13.1, but is planned to become opt-in from PyO3 0.14.0. [#1347](https://github.com/PyO3/pyo3/pull/1347)
 - Add support for cross-compiling to Windows without needing `PYO3_CROSS_INCLUDE_DIR`. [#1350](https://github.com/PyO3/pyo3/pull/1350)
 
 ### Changed

Cargo.toml

@@ -33,26 +33,36 @@ assert_approx_eq = "1.1.0"
 trybuild = "1.0.23"
 rustversion = "1.0"
 proptest = { version = "0.10.1", default-features = false, features = ["std"] }
+# features needed to run the PyO3 test suite
+pyo3 = { path = ".", default-features = false, features = ["macros", "auto-initialize"] }
 
 [features]
-default = ["macros"]
-macros = ["ctor", "indoc", "inventory", "paste", "pyo3-macros", "unindent"]
+default = ["macros", "auto-initialize"]
+
+# Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc.
+macros = ["pyo3-macros", "ctor", "indoc", "inventory", "paste", "unindent"]
+
+# Use this feature when building an extension module.
+# It tells the linker to keep the python symbols unresolved,
+# so that the module can also be used with statically linked python interpreters.
+extension-module = []
+
 # Use the Python limited API. See https://www.python.org/dev/peps/pep-0384/ for more.
 abi3 = []
+
 # With abi3, we can manually set the minimum Python version.
 abi3-py36 = ["abi3-py37"]
 abi3-py37 = ["abi3-py38"]
 abi3-py38 = ["abi3-py39"]
 abi3-py39 = ["abi3"]
 
+# Changes `Python::with_gil` and `Python::acquire_gil` to automatically initialize the
+# Python interpreter if needed.
+auto-initialize = []
+
 # Optimizes PyObject to Vec conversion and so on.
 nightly = []
 
-# Use this feature when building an extension module.
-# It tells the linker to keep the python symbols unresolved,
-# so that the module can also be used with statically linked python interpreters.
-extension-module = []
-
 [workspace]
 members = [
     "pyo3-macros",

README.md

@@ -104,8 +104,9 @@ If you want your Rust application to create a Python interpreter internally and
 use it to run Python code, add `pyo3` to your `Cargo.toml` like this:
 
 ```toml
-[dependencies]
-pyo3 = "0.13.0"
+[dependencies.pyo3]
+version = "0.13.0"
+features = ["auto-initialize"]
 ```
 
 Example program displaying the value of `sys.version` and the current user name:

build.rs

@@ -749,6 +749,10 @@ fn configure(interpreter_config: &InterpreterConfig) -> Result<()> {
         }
     }
 
+    if interpreter_config.shared {
+        println!("cargo:rustc-cfg=Py_SHARED");
+    }
+
     if interpreter_config.version.implementation == PythonInterpreterKind::PyPy {
         println!("cargo:rustc-cfg=PyPy");
     };
@@ -883,5 +887,11 @@ fn main() -> Result<()> {
         }
     }
 
+    // TODO: this is a hack to workaround compile_error! warnings about auto-initialize on PyPy
+    // Once cargo's `resolver = "2"` is stable (~ MSRV Rust 1.52), remove this.
+    if env::var_os("PYO3_CI").is_some() {
+        println!("cargo:rustc-cfg=__pyo3_ci");
+    }
+
     Ok(())
 }

guide/src/SUMMARY.md

@@ -16,6 +16,7 @@
 - [GIL, mutability and object types](types.md)
 - [Parallelism](parallelism.md)
 - [Debugging](debugging.md)
+- [Features Reference](features.md)
 - [Advanced Topics](advanced.md)
 - [Building and Distribution](building_and_distribution.md)
 - [PyPy support](pypy.md)

guide/src/advanced.md

@@ -15,9 +15,3 @@ The caveat to these "owned references" is that Rust references do not normally c
 For most use cases this behaviour is invisible. Occasionally, however, users may need to clear memory usage sooner than PyO3 usually does. PyO3 exposes this functionality with the  the `GILPool` struct. When a `GILPool` is dropped, ***all*** owned references created after the `GILPool` was created will be cleared.
 
 The unsafe function `Python::new_pool` allows you to create a new `GILPool`. When doing this, you must be very careful to ensure that once the `GILPool` is dropped you do not retain access any owned references created after the `GILPool` was created.
-
-## The `nightly` feature
-
-The `pyo3/nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:
-- `FromPyObject` for `Vec` and `[T;N]` can perform a `memcpy` when the object is a `PyBuffer`
-- `ToBorrowedObject` can skip a reference count increase when the provided object is a Python native type.

guide/src/building_and_distribution.md

@@ -42,7 +42,7 @@ There are two ways to distribute your module as a Python package: The old, [setu
 
 By default, Python extension modules can only be used with the same Python version they were compiled against -- if you build an extension module with Python 3.5, you can't import it using Python 3.8. [PEP 384](https://www.python.org/dev/peps/pep-0384/) introduced the idea of the limited Python API, which would have a stable ABI enabling extension modules built with it to be used against multiple Python versions. This is also known as `abi3`.
 
-Note that [maturin] >= 0.9.0 or [setuptools-rust] >= 0.12.0 is going to support `abi3` wheels.
+Note that [maturin] >= 0.9.0 or [setuptools-rust] >= 0.11.4 support `abi3` wheels.
 See the [corresponding](https://github.com/PyO3/maturin/pull/353) [PRs](https://github.com/PyO3/setuptools-rust/pull/82) for more.
 
 There are three steps involved in making use of `abi3` when building Python packages as wheels:

guide/src/features.md

@@ -0,0 +1,64 @@
+# Features Reference
+
+PyO3 provides a number of Cargo features to customise functionality. This chapter of the guide provides detail on each of them.
+
+By default, the `macros` and `auto-initialize` features are enabled.
+
+## Features for extension module authors
+
+### `extension-module`
+
+This feature is required when building a Python extension module using PyO3.
+
+It tells PyO3's build script to skip linking against `libpython.so` on Unix platforms, where this must not be done.
+
+See the [building and distribution](building_and_distribution.md#linking) section for further detail.
+
+### `abi3`
+
+This feature is used when building Python extension modules to create wheels which are compatible with multiple Python versions.
+
+It restricts PyO3's API to a subset of the full Python API which is guaranteed by [PEP 384](https://www.python.org/dev/peps/pep-0384/) to be forwards-compatible with future Python versions.
+
+See the [building and distribution](building_and_distribution.md#py_limited_apiabi3) section for further detail.
+
+### `abi3-py36` / `abi3-py37` / `abi3-py38` / `abi3-py39`
+
+These features are an extension of the `abi3` feature to specify the exact minimum Python version which the multiple-version-wheel will support.
+
+See the [building and distribution](building_and_distribution.md#minimum-python-version-for-abi3) section for further detail.
+
+## Features for embedding Python in Rust
+
+### `auto-initalize`
+
+This feature changes [`Python::with_gil`](https://docs.rs/pyo3/latest/pyo3/struct.Python.html#method.with_gil) and [`Python::acquire_gil`](https://docs.rs/pyo3/latest/pyo3/struct.Python.html#method.acquire_gil) to automatically initialize a Python interpreter (by calling [`prepare_freethreaded_python`](https://docs.rs/pyo3/latest/pyo3/fn.prepare_freethreaded_python.html)) if needed.
+
+This feature is not needed for extension modules, but for compatibility it is enabled by default until at least the PyO3 0.14 release.
+
+> This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
+
+## Advanced Features
+
+### `macros`
+
+This feature enables a dependency on the `pyo3-macros` crate, which provides the procedural macros portion of PyO3's API:
+
+- `#[pymodule]`
+- `#[pyfunction]`
+- `#[pyclass]`
+- `#[pymethods]`
+- `#[pyproto]`
+- `#[derive(FromPyObject)]`
+
+It also provides the `py_run!` macro.
+
+These macros require a number of dependencies which may not be needed by users who just need PyO3 for Python FFI. Disabling this feature enables faster builds for those users, as these dependencies will not be built if this feature is disabled.
+
+> This feature is enabled by default. To disable it, set `default-features = false` for the `pyo3` entry in your Cargo.toml.
+
+### `nightly`
+
+The `nightly` feature needs the nightly Rust compiler. This allows PyO3 to use Rust's unstable specialization feature to apply the following optimizations:
+- `FromPyObject` for `Vec` and `[T;N]` can perform a `memcpy` when the object supports the Python buffer protocol.
+- `ToBorrowedObject` can skip a reference count increase when the provided object is a Python native type.

src/gil.rs

@@ -3,11 +3,11 @@
 //! Interaction with python's global interpreter lock
 
 use crate::{ffi, internal_tricks::Unsendable, Python};
-use parking_lot::{const_mutex, Mutex};
+use parking_lot::{const_mutex, Mutex, Once};
 use std::cell::{Cell, RefCell};
-use std::{mem::ManuallyDrop, ptr::NonNull, sync};
+use std::{mem::ManuallyDrop, ptr::NonNull};
 
-static START: sync::Once = sync::Once::new();
+static START: Once = Once::new();
 
 thread_local! {
     /// This is a internal counter in pyo3 monitoring whether this thread has the GIL.
@@ -45,16 +45,20 @@ pub(crate) fn gil_is_acquired() -> bool {
 /// If both the Python interpreter and Python threading are already initialized,
 /// this function has no effect.
 ///
+/// # Availability
+///
+/// This function is only available when linking against Python distributions that contain a
+/// shared library.
+///
+/// This function is not available on PyPy.
+///
 /// # Panic
 /// If the Python interpreter is initialized but Python threading is not,
 /// a panic occurs.
 /// It is not possible to safely access the Python runtime unless the main
 /// thread (the thread which originally initialized Python) also initializes
 /// threading.
-///
-/// When writing an extension module, the `#[pymodule]` macro
-/// will ensure that Python threading is initialized.
-///
+#[cfg(all(Py_SHARED, not(PyPy)))]
 pub fn prepare_freethreaded_python() {
     // Protect against race conditions when Python is not yet initialized
     // and multiple threads concurrently call 'prepare_freethreaded_python()'.
@@ -72,34 +76,29 @@ pub fn prepare_freethreaded_python() {
             // Note that the 'main thread' notion in Python isn't documented properly;
             // and running Python without one is not officially supported.
 
-            // PyPy does not support the embedding API
-            #[cfg(not(PyPy))]
-            {
-                ffi::Py_InitializeEx(0);
-
-                // Make sure Py_Finalize will be called before exiting.
-                extern "C" fn finalize() {
-                    unsafe {
-                        if ffi::Py_IsInitialized() != 0 {
-                            ffi::PyGILState_Ensure();
-                            ffi::Py_Finalize();
-                        }
+            ffi::Py_InitializeEx(0);
+
+            // Make sure Py_Finalize will be called before exiting.
+            extern "C" fn finalize() {
+                unsafe {
+                    if ffi::Py_IsInitialized() != 0 {
+                        ffi::PyGILState_Ensure();
+                        ffi::Py_Finalize();
                     }
                 }
-                libc::atexit(finalize);
             }
+            libc::atexit(finalize);
 
             // > Changed in version 3.7: This function is now called by Py_Initialize(), so you don’t have
             // > to call it yourself anymore.
             #[cfg(not(Py_3_7))]
             if ffi::PyEval_ThreadsInitialized() == 0 {
                 ffi::PyEval_InitThreads();
             }
-            // PyEval_InitThreads() will acquire the GIL,
-            // but we don't want to hold it at this point
+
+            // Py_InitializeEx() will acquire the GIL, but we don't want to hold it at this point
             // (it's not acquired in the other code paths)
             // So immediately release the GIL:
-            #[cfg(not(PyPy))]
             let _thread_state = ffi::PyEval_SaveThread();
             // Note that the PyThreadState returned by PyEval_SaveThread is also held in TLS by the Python runtime,
             // and will be restored by PyGILState_Ensure.
@@ -137,7 +136,51 @@ impl GILGuard {
     /// If PyO3 does not yet have a `GILPool` for tracking owned PyObject references, then this
     /// new `GILGuard` will also contain a `GILPool`.
     pub(crate) fn acquire() -> GILGuard {
-        prepare_freethreaded_python();
+        // Maybe auto-initialize the GIL:
+        //  - If auto-initialize feature set and supported, try to initalize the interpreter.
+        //  - If the auto-initialize feature is set but unsupported, emit hard errors only when
+        //    the extension-module feature is not activated - extension modules don't care about
+        //    auto-initialize so this avoids breaking existing builds.
+        //  - Otherwise, just check the GIL is initialized.
+        cfg_if::cfg_if! {
+            if #[cfg(all(feature = "auto-initialize", Py_SHARED, not(PyPy)))] {
+                prepare_freethreaded_python();
+            } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), not(Py_SHARED), not(__pyo3_ci)))] {
+                compile_error!(concat!(
+                    "The `auto-initialize` feature is not supported when linking Python ",
+                    "statically instead of with a shared library.\n\n",
+                    "Please disable the `auto-initialize` feature, for example by entering the following ",
+                    "in your cargo.toml:\n\n",
+                    "    pyo3 = { version = \"0.13.0\", default-features = false }\n\n",
+                    "Alternatively, compile PyO3 using a Python distribution which contains a shared ",
+                    "libary."
+                ));
+            } else if #[cfg(all(feature = "auto-initialize", not(feature = "extension-module"), PyPy, not(__pyo3_ci)))] {
+                compile_error!(concat!(
+                    "The `auto-initialize` feature is not supported by PyPy.\n\n",
+                    "Please disable the `auto-initialize` feature, for example by entering the following ",
+                    "in your cargo.toml:\n\n",
+                    "    pyo3 = { version = \"0.13.0\", default-features = false }",
+                ));
+            } else {
+                // extension module feature enabled and PyPy or static linking
+                // OR auto-initialize feature not enabled
+                START.call_once_force(|_| unsafe {
+                    // Use call_once_force because if there is a panic because the interpreter is not
+                    // initialized, it's fine for the user to initialize the interpreter and retry.
+                    assert_ne!(
+                        ffi::Py_IsInitialized(),
+                        0,
+                        "The Python interpreter is not initalized and the `auto-initialize` feature is not enabled."
+                    );
+                    assert_ne!(
+                        ffi::PyEval_ThreadsInitialized(),
+                        0,
+                        "Python threading is not initalized and the `auto-initialize` feature is not enabled."
+                    );
+                });
+            }
+        }
 
         let gstate = unsafe { ffi::PyGILState_Ensure() }; // acquire GIL
 

src/lib.rs

@@ -114,8 +114,9 @@
 //! Add `pyo3` to your `Cargo.toml`:
 //!
 //! ```toml
-//! [dependencies]
-//! pyo3 = "0.13.0"
+//! [dependencies.pyo3]
+//! version = "0.13.0"
+//! features = ["auto-initialize"]
 //! ```
 //!
 //! Example program displaying the value of `sys.version`:
@@ -145,12 +146,14 @@ pub use crate::conversion::{
     ToBorrowedObject, ToPyObject,
 };
 pub use crate::err::{PyDowncastError, PyErr, PyErrArguments, PyResult};
+#[cfg(all(Py_SHARED, not(PyPy)))]
+pub use crate::gil::prepare_freethreaded_python;
 pub use crate::gil::{GILGuard, GILPool};
 pub use crate::instance::{Py, PyNativeType, PyObject};
 pub use crate::pycell::{PyCell, PyRef, PyRefMut};
 pub use crate::pyclass::PyClass;
 pub use crate::pyclass_init::PyClassInitializer;
-pub use crate::python::{prepare_freethreaded_python, Python, PythonVersionInfo};
+pub use crate::python::{Python, PythonVersionInfo};
 pub use crate::type_object::{type_flags, PyTypeInfo};
 // Since PyAny is as important as PyObject, we expose it to the top level.
 pub use crate::types::PyAny;