WIP: Generalize the rpath API for relocatable Python installations

geofft · geofft · commit e9df01c4d3c2 · 2025-02-03T15:40:38.000-05:00
Relocatable Python installations like python-build-standalone have a similar need for an rpath as the macOS framework builds in Xcode. Furthermore, the platform on which you build your code is not really a function of the code itself. This change generalizes the recent `add_python_framework_link_args()` from PyO3#4833 to `add_python_link_args()`, which also does the right thing on python-build-standalone, and encourages everyone to set up a build.rs file that invokes it, whether or not they need it on their current machine.
diff --git a/build.rs b/build.rs
@@ -2,7 +2,7 @@ use std::env;
 
 use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result};
 use pyo3_build_config::{
-    add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig,
+    add_python_link_args, bail, print_feature_cfgs, InterpreterConfig,
 };
 
 fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> {
@@ -44,8 +44,8 @@ fn configure_pyo3() -> Result<()> {
     // Emit cfgs like `invalid_from_utf8_lint`
     print_feature_cfgs();
 
-    // Make `cargo test` etc work on macOS with Xcode bundled Python
-    add_python_framework_link_args();
+    // Make `cargo test` etc work on non-global Python installations
+    add_python_link_args();
 
     Ok(())
 }
diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md
@@ -120,6 +120,9 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { {{#PYO3_CRATE_VERSION}}, features = ["extension-module"] }
+
+[build-dependencies]
+pyo3-build-config = { {{#PYO3_CRATE_VERSION} }
 ```
 
 ## pyproject.toml
@@ -141,6 +144,17 @@ classifiers = [
 ]
 ```
 
+## build.rs
+
+Finally, in order to make `cargo test` work correctly, you should create
+a `build.rs` file with the following contents:
+
+```
+fn main() {
+    pyo3_build_config::add_python_link_args();
+}
+```
+
 ## Running code
 
 After this you can setup Rust code to be available in Python as below; for example, you can place this code in `src/lib.rs`:
diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs
@@ -167,8 +167,12 @@ pub struct InterpreterConfig {
     ///
     /// Serialized to multiple `extra_build_script_line` values.
     pub extra_build_script_lines: Vec<String>,
+
     /// macOS Python3.framework requires special rpath handling
     pub python_framework_prefix: Option<String>,
+
+    /// Relocatable Python installations require special rpath handling
+    pub relocatable: bool,
 }
 
 impl InterpreterConfig {
@@ -265,6 +269,7 @@ print("calcsize_pointer", struct.calcsize("P"))
 print("mingw", get_platform().startswith("mingw"))
 print("ext_suffix", get_config_var("EXT_SUFFIX"))
 print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
+print_if_set("python_build_standalone", get_config_var("PYTHON_BUILD_STANDALONE"))
 "#;
         let output = run_python_script(interpreter.as_ref(), SCRIPT)?;
         let map: HashMap<String, String> = parse_script_output(&output);
@@ -315,6 +320,13 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
             _ => panic!("Unknown Py_GIL_DISABLED value"),
         };
 
+        let relocatable = match map.get("python_build_standalone").map(String::as_str) {
+            None => false,
+            Some("0") => false,
+            Some("1") => true,
+            _ => panic!("Unknown PYTHON_BUILD_STANDALONE value"),
+        };
+
         let lib_name = if cfg!(windows) {
             default_lib_name_windows(
                 version,
@@ -365,6 +377,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix,
+            relocatable,
         })
     }
 
@@ -410,6 +423,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
             Some(value) => value == "1",
             None => false,
         };
+        let relocatable = match sysconfigdata.get_value("python_build_standalone") {
+            Some(value) => value == "1",
+            None => false,
+        };
         let lib_name = Some(default_lib_name_unix(
             version,
             implementation,
@@ -434,6 +451,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix,
+            relocatable,
         })
     }
 
@@ -511,6 +529,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
         let mut suppress_build_script_link_lines = None;
         let mut extra_build_script_lines = vec![];
         let mut python_framework_prefix = None;
+        let mut relocatable = None;
 
         for (i, line) in lines.enumerate() {
             let line = line.context("failed to read line from config")?;
@@ -540,6 +559,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
                     extra_build_script_lines.push(value.to_string());
                 }
                 "python_framework_prefix" => parse_value!(python_framework_prefix, value),
+                "relocatable" => parse_value!(relocatable, value),
                 unknown => warn!("unknown config key `{}`", unknown),
             }
         }
@@ -571,6 +591,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
             suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false),
             extra_build_script_lines,
             python_framework_prefix,
+            relocatable: relocatable.unwrap_or(false),
         })
     }
 
@@ -663,12 +684,13 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED"))
         write_option_line!(executable)?;
         write_option_line!(pointer_width)?;
         write_line!(build_flags)?;
-        write_option_line!(python_framework_prefix)?;
         write_line!(suppress_build_script_link_lines)?;
         for line in &self.extra_build_script_lines {
             writeln!(writer, "extra_build_script_line={}", line)
                 .context("failed to write extra_build_script_line")?;
         }
+        write_option_line!(python_framework_prefix)?;
+        write_line!(relocatable)?;
         Ok(())
     }
 
@@ -1601,7 +1623,10 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result<In
         build_flags: BuildFlags::default(),
         suppress_build_script_link_lines: false,
         extra_build_script_lines: vec![],
+        // Because this is only used for extensions, we know we do not
+        // need any rpath handling.
         python_framework_prefix: None,
+        relocatable: false,
     })
 }
 
@@ -1644,7 +1669,10 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result<Interpre
         build_flags: BuildFlags::default(),
         suppress_build_script_link_lines: false,
         extra_build_script_lines: vec![],
+        // Because this is only used for extensions, we know we do not
+        // need any rpath handling.
         python_framework_prefix: None,
+        relocatable: false,
     })
 }
 
@@ -2028,6 +2056,7 @@ mod tests {
             suppress_build_script_link_lines: true,
             extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()],
             python_framework_prefix: None,
+            relocatable: false,
         };
         let mut buf: Vec<u8> = Vec::new();
         config.to_writer(&mut buf).unwrap();
@@ -2057,6 +2086,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: true,
         };
         let mut buf: Vec<u8> = Vec::new();
         config.to_writer(&mut buf).unwrap();
@@ -2079,6 +2109,7 @@ mod tests {
             suppress_build_script_link_lines: true,
             extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()],
             python_framework_prefix: None,
+            relocatable: false,
         };
         let mut buf: Vec<u8> = Vec::new();
         config.to_writer(&mut buf).unwrap();
@@ -2106,6 +2137,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         )
     }
@@ -2129,6 +2161,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         )
     }
@@ -2232,6 +2265,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
     }
@@ -2262,6 +2296,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
 
@@ -2289,6 +2324,37 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
+            }
+        );
+    }
+
+    #[test]
+    fn config_from_sysconfigdata_relocatable() {
+        let mut sysconfigdata = Sysconfigdata::new();
+        sysconfigdata.insert("SOABI", "cpython-37m-x86_64-linux-gnu");
+        sysconfigdata.insert("VERSION", "3.7");
+        sysconfigdata.insert("Py_ENABLE_SHARED", "1");
+        sysconfigdata.insert("LIBDIR", "/home/xyz/Downloads/python/lib");
+        sysconfigdata.insert("LDVERSION", "3.7m");
+        sysconfigdata.insert("SIZEOF_VOID_P", "8");
+        sysconfigdata.insert("PYTHON_BUILD_STANDALONE", "1");
+        assert_eq!(
+            InterpreterConfig::from_sysconfigdata(&sysconfigdata).unwrap(),
+            InterpreterConfig {
+                abi3: false,
+                build_flags: BuildFlags::from_sysconfigdata(&sysconfigdata),
+                pointer_width: Some(64),
+                executable: None,
+                implementation: PythonImplementation::CPython,
+                lib_dir: Some("/home/xyz/Downloads/python/lib".into()),
+                lib_name: Some("python3.7m".into()),
+                shared: true,
+                version: PythonVersion::PY37,
+                suppress_build_script_link_lines: false,
+                extra_build_script_lines: vec![],
+                python_framework_prefix: None,
+                relocatable: true,
             }
         );
     }
@@ -2313,6 +2379,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
     }
@@ -2337,6 +2404,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
     }
@@ -2372,6 +2440,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relcoatable: false,
             }
         );
     }
@@ -2407,6 +2476,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
     }
@@ -2442,6 +2512,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable; false,
             }
         );
     }
@@ -2479,6 +2550,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         );
     }
@@ -2827,6 +2899,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
 
         config
@@ -2850,6 +2923,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
 
         assert!(config
@@ -2915,6 +2989,7 @@ mod tests {
                 suppress_build_script_link_lines: false,
                 extra_build_script_lines: vec![],
                 python_framework_prefix: None,
+                relocatable: false,
             }
         )
     }
@@ -3040,6 +3115,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
         assert_eq!(
             interpreter_config.build_script_outputs(),
@@ -3080,6 +3156,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
 
         assert_eq!(
@@ -3128,6 +3205,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
 
         assert_eq!(
@@ -3162,6 +3240,7 @@ mod tests {
             suppress_build_script_link_lines: false,
             extra_build_script_lines: vec![],
             python_framework_prefix: None,
+            relocatable: false,
         };
 
         assert_eq!(
diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs
