Adds a new target 'pybind11::static' that allows compiling pybind11 as a static library and linking against it.

This consists of the following changes:

1. CMake infrastructure to support exporting and installing this target, with an option to disable it.
2. Defining some new macros to support this dual mode compilation.
3. Creating one .cc file that is compiled into the new static library. The rest of the code could be refactored in subsequent PRs.
4. Creating tests that compiling against the new target works.

Co-authored-by: Ciro Santilli <[email protected]>

Author: 2 people

.github/CONTRIBUTING.md

@@ -116,7 +116,8 @@ The valid options are:
 * `-DCMAKE_BUILD_TYPE`: Release, Debug, MinSizeRel, RelWithDebInfo
 * `-DPYBIND11_FINDPYTHON=ON`: Use CMake 3.12+'s FindPython instead of the
   classic, deprecated, custom FindPythonLibs
-* `-DPYBIND11_NOPYTHON=ON`: Disable all Python searching (disables tests)
+* `-DPYBIND11_NOPYTHON=ON`: Disable all Python searching (disables tests and the static library target)
+* `-DPYBIND11_BUILD_STATIC_LIB=OFF`: Don't build a static library target.
 * `-DBUILD_TESTING=ON`: Enable the tests
 * `-DDOWNLOAD_CATCH=ON`: Download catch to build the C++ tests
 * `-DDOWNLOAD_EIGEN=ON`: Download Eigen for the NumPy tests

CMakeLists.txt

@@ -91,6 +91,8 @@ endif()
 option(PYBIND11_INSTALL "Install pybind11 header files?" ${PYBIND11_MASTER_PROJECT})
 option(PYBIND11_TEST "Build pybind11 test suite?" ${PYBIND11_MASTER_PROJECT})
 option(PYBIND11_NOPYTHON "Disable search for Python" OFF)
+option(PYBIND11_BUILD_STATIC_LIB
+       "Create a static library target for pybind11 that is not header-only (default)" ON)
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
@@ -107,6 +109,7 @@ cmake_dependent_option(PYBIND11_FINDPYTHON "Force new FindPython" OFF
 set(PYBIND11_HEADERS
     include/pybind11/detail/class.h
     include/pybind11/detail/common.h
+    include/pybind11/detail/common-inl.h
     include/pybind11/detail/descr.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
@@ -133,6 +136,30 @@ set(PYBIND11_HEADERS
     include/pybind11/stl_bind.h
     include/pybind11/stl/filesystem.h)
 
+file(GLOB_RECURSE PYBIND11_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/*.cc)
+# `embed.cc` is the TU to support embedding a python interpreter. But PyPy does not
+# support embedding, so remove it from sources.
+if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy")
+  # Pypy does not support embedding.
+  list(REMOVE_ITEM PYBIND11_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/src/embed.cc)
+endif()
+
+# If we're not configured to be header-only, we need to build a shared library
+# and define the pybind11::lib target.
+if(PYBIND11_BUILD_STATIC_LIB)
+  add_library(pybind11_static STATIC ${PYBIND11_SOURCES} ${PYBIND11_HEADERS})
+  set_property(TARGET pybind11_static PROPERTY POSITION_INDEPENDENT_CODE ON)
+  target_compile_definitions(pybind11_static PUBLIC -DPYBIND11_AS_STATIC_LIBRARY=1)
+  add_library(pybind11::static ALIAS pybind11_static)
+  target_link_libraries(pybind11_static PRIVATE pybind11::pybind11)
+  target_link_libraries(pybind11_static PUBLIC pybind11::headers)
+  install(
+    TARGETS pybind11_static
+    EXPORT pybind11::static
+    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+endif()
+
 # Compare with grep and warn if mismatched
 if(PYBIND11_MASTER_PROJECT AND NOT CMAKE_VERSION VERSION_LESS 3.12)
   file(
@@ -259,6 +286,13 @@ if(PYBIND11_INSTALL)
   endif()
 
   install(TARGETS pybind11_headers EXPORT "${PYBIND11_EXPORT_NAME}")
+  if(PYBIND11_BUILD_STATIC_LIB)
+    install(
+      TARGETS pybind11_static
+      EXPORT "${PYBIND11_EXPORT_NAME}"
+      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
+  endif()
 
   install(
     EXPORT "${PYBIND11_EXPORT_NAME}"
@@ -304,6 +338,10 @@ else()
   endif()
 endif()
 
+if(_pybind11_nopython AND PYBIND11_BUILD_STATIC_LIB)
+  message(FATAL_ERROR "Cannot build static library in NOPYTHON mode")
+endif()
+
 # Better symmetry with find_package(pybind11 CONFIG) mode.
 if(NOT PYBIND11_MASTER_PROJECT)
   set(pybind11_FOUND

MANIFEST.in

@@ -1,4 +1,5 @@
 recursive-include pybind11/include/pybind11 *.h
+recursive-include src *.cc
 recursive-include pybind11 *.py
 recursive-include pybind11 py.typed
 include pybind11/share/cmake/pybind11/*.cmake

docs/Doxyfile

@@ -11,6 +11,7 @@ XML_PROGRAMLISTING     = YES
 MACRO_EXPANSION        = YES
 EXPAND_ONLY_PREDEF     = YES
 EXPAND_AS_DEFINED      = PYBIND11_RUNTIME_EXCEPTION
+EXCLUDE_PATTERNS       = *-inl.h
 
 ALIASES                = "rst=\verbatim embed:rst"
 ALIASES               += "endrst=\endverbatim"

docs/compiling.rst

@@ -482,6 +482,9 @@ available in all modes. The targets provided are:
    ``pybind11::opt_size``
      ``/Os`` for MSVC, ``-Os`` for other compilers. Does nothing for debug builds.
 
+   ``pybind11::static``
+     Statically link against libpybind11, and configure the pybind11 headers to not provide definitions that they would provide in the default header-only mode.
+
 Two helper functions are also provided:
 
     ``pybind11_strip(target)``

include/pybind11/detail/common-inl.h

@@ -0,0 +1,25 @@
+/*
+    pybind11/detail/common-inl.h -- Basic macros definitions
+
+    Copyright (c) 2016-2022 Wenzel Jakob <[email protected]>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+#include "pybind11/detail/common.h"
+
+PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE)
+
+PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void pybind11_fail(const char *reason) {
+    assert(!PyErr_Occurred());
+    throw std::runtime_error(reason);
+}
+PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void pybind11_fail(const std::string &reason) {
+    assert(!PyErr_Occurred());
+    throw std::runtime_error(reason);
+}
+
+PYBIND11_INLINE error_scope::error_scope() { PyErr_Fetch(&type, &value, &trace); }
+PYBIND11_INLINE error_scope::~error_scope() { PyErr_Restore(type, value, trace); };
+
+PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

include/pybind11/detail/common.h

@@ -117,17 +117,29 @@
 #    define PYBIND11_NOINLINE_DISABLED
 #endif
 
-// The PYBIND11_NOINLINE macro is for function DEFINITIONS.
-// In contrast, FORWARD DECLARATIONS should never use this macro:
-// https://stackoverflow.com/questions/9317473/forward-declaration-of-inline-functions
+// PYBIND11_INLINE should be used for function definitions in '-inl' files, so they
+// can be made non-inline when compiles as a static library.
+#if defined(PYBIND11_AS_STATIC_LIBRARY)
+#    define PYBIND11_INLINE
+#else
+#    define PYBIND11_INLINE inline
+#endif
+
 #if defined(PYBIND11_NOINLINE_DISABLED) // Option for maximum portability and experimentation.
-#    define PYBIND11_NOINLINE inline
+#    define PYBIND11_NOINLINE_ATTR
 #elif defined(_MSC_VER)
-#    define PYBIND11_NOINLINE __declspec(noinline) inline
+#    define PYBIND11_NOINLINE_ATTR __declspec(noinline)
 #else
-#    define PYBIND11_NOINLINE __attribute__((noinline)) inline
+#    define PYBIND11_NOINLINE_ATTR __attribute__((noinline))
 #endif
 
+// The PYBIND11_NOINLINE macro is for function DEFINITIONS.
+// In contrast, FORWARD DECLARATIONS should never use this macro:
+// https://stackoverflow.com/questions/9317473/forward-declaration-of-inline-functions
+// This macro shouldn't be used in '-inl' files. Instead, use `PYBIND11_NOINLINE_ATTR
+// PYBIND11_INLINE`.
+#define PYBIND11_NOINLINE PYBIND11_NOINLINE_ATTR inline
+
 #if defined(__MINGW32__)
 // For unknown reasons all PYBIND11_DEPRECATED member trigger a warning when declared
 // whether it is used or not
@@ -936,14 +948,8 @@ PYBIND11_RUNTIME_EXCEPTION(cast_error, PyExc_RuntimeError) /// Thrown when pybin
                                                            /// casting error
 PYBIND11_RUNTIME_EXCEPTION(reference_cast_error, PyExc_RuntimeError) /// Used internally
 
-[[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const char *reason) {
-    assert(!PyErr_Occurred());
-    throw std::runtime_error(reason);
-}
-[[noreturn]] PYBIND11_NOINLINE void pybind11_fail(const std::string &reason) {
-    assert(!PyErr_Occurred());
-    throw std::runtime_error(reason);
-}
+[[noreturn]] void pybind11_fail(const char *reason);
+[[noreturn]] void pybind11_fail(const std::string &reason);
 
 template <typename T, typename SFINAE = void>
 struct format_descriptor {};
@@ -992,10 +998,10 @@ constexpr const char
 /// RAII wrapper that temporarily clears any Python error state
 struct error_scope {
     PyObject *type, *value, *trace;
-    error_scope() { PyErr_Fetch(&type, &value, &trace); }
+    error_scope();
     error_scope(const error_scope &) = delete;
     error_scope &operator=(const error_scope &) = delete;
-    ~error_scope() { PyErr_Restore(type, value, trace); }
+    ~error_scope();
 };
 
 /// Dummy destructor wrapper that can be used to expose classes with a private destructor
@@ -1185,3 +1191,7 @@ constexpr inline bool silence_msvc_c4127(bool cond) { return cond; }
 
 PYBIND11_NAMESPACE_END(detail)
 PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)
+
+#ifndef PYBIND11_AS_STATIC_LIBRARY
+#    include "common-inl.h"
+#endif

setup.py

@@ -128,6 +128,7 @@ def remove_output(*sources: str) -> Iterator[None]:
             "-DBUILD_TESTING=OFF",
             "-DPYBIND11_NOPYTHON=ON",
             "-Dprefix_for_pc_file=${pcfiledir}/../../",
+            "-DPYBIND11_BUILD_STATIC_LIB=OFF",
         ]
         if "CMAKE_ARGS" in os.environ:
             fcommand = [

src/detail/common.cc

@@ -0,0 +1 @@
+#include "pybind11/detail/common-inl.h"

tests/extra_python_package/test_files.py

@@ -48,6 +48,7 @@
 detail_headers = {
     "include/pybind11/detail/class.h",
     "include/pybind11/detail/common.h",
+    "include/pybind11/detail/common-inl.h",
     "include/pybind11/detail/descr.h",
     "include/pybind11/detail/init.h",
     "include/pybind11/detail/internals.h",

tests/test_cmake_build/CMakeLists.txt

@@ -37,7 +37,8 @@ function(pybind11_add_build_test name)
     "${CMAKE_CURRENT_SOURCE_DIR}/${name}"
     "${CMAKE_CURRENT_BINARY_DIR}/${name}"
     --build-config
-    Release
+    "$<$<CONFIG:DEBUG>:Debug>"
+    "$<$<CONFIG:RELEASE>:Release>"
     --build-noclean
     --build-generator
     ${CMAKE_GENERATOR}
@@ -59,6 +60,9 @@ possibly_uninitialized(PYTHON_MODULE_EXTENSION Python_INTERPRETER_ID)
 
 pybind11_add_build_test(subdirectory_function)
 pybind11_add_build_test(subdirectory_target)
+if(PYBIND11_BUILD_STATIC_LIB)
+  pybind11_add_build_test(subdirectory_static)
+endif()
 if("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy")
   message(STATUS "Skipping embed test on PyPy")
 else()
@@ -67,11 +71,15 @@ endif()
 
 if(PYBIND11_INSTALL)
   add_custom_target(
-    mock_install ${CMAKE_COMMAND} "-DCMAKE_INSTALL_PREFIX=${pybind11_BINARY_DIR}/mock_install" -P
-                 "${pybind11_BINARY_DIR}/cmake_install.cmake")
+    mock_install ${CMAKE_COMMAND} "-DCMAKE_INSTALL_PREFIX=${pybind11_BINARY_DIR}/mock_install"
+    "-DBUILD_TYPE=" "$<$<CONFIG:DEBUG>:Debug>" "$<$<CONFIG:RELEASE>:Release>" -P "${pybind11_BINARY_DIR}/cmake_install.cmake")
 
   pybind11_add_build_test(installed_function INSTALL)
   pybind11_add_build_test(installed_target INSTALL)
+  if(PYBIND11_BUILD_STATIC_LIB)
+    pybind11_add_build_test(installed_static INSTALL)
+    add_dependencies(mock_install pybind11::static)
+  endif()
   if(NOT ("${PYTHON_MODULE_EXTENSION}" MATCHES "pypy" OR "${Python_INTERPRETER_ID}" STREQUAL "PyPy"
          ))
     pybind11_add_build_test(installed_embed INSTALL)

tests/test_cmake_build/installed_static/CMakeLists.txt

@@ -0,0 +1,46 @@
+cmake_minimum_required(VERSION 3.4)
+
+# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with
+# some versions of VS that have a patched CMake 3.11. This forces us to emulate
+# the behavior using the following workaround:
+if(${CMAKE_VERSION} VERSION_LESS 3.18)
+  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
+else()
+  cmake_policy(VERSION 3.18)
+endif()
+
+project(test_installed_static CXX)
+
+find_package(pybind11 CONFIG REQUIRED)
+message(STATUS "Found pybind11 v${pybind11_VERSION}: ${pybind11_INCLUDE_DIRS}")
+
+add_library(test_installed_static MODULE ../main.cpp)
+
+target_link_libraries(test_installed_static PRIVATE pybind11::module pybind11::static)
+set_target_properties(test_installed_static PROPERTIES OUTPUT_NAME test_cmake_build)
+
+# Make sure result is, for example, test_installed_static.so, not libtest_installed_static.dylib
+pybind11_extension(test_installed_static)
+
+# Do not treat includes from IMPORTED target as SYSTEM (Python headers in pybind11::module).
+# This may be needed to resolve header conflicts, e.g. between Python release and debug headers.
+set_target_properties(test_installed_static PROPERTIES NO_SYSTEM_FROM_IMPORTED ON)
+
+if(DEFINED Python_EXECUTABLE)
+  set(_Python_EXECUTABLE "${Python_EXECUTABLE}")
+elseif(DEFINED PYTHON_EXECUTABLE)
+  set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
+else()
+  message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)")
+endif()
+
+add_custom_target(
+  check_installed_static
+  ${CMAKE_COMMAND}
+  -E
+  env
+  PYTHONPATH=$<TARGET_FILE_DIR:test_installed_static>
+  ${_Python_EXECUTABLE}
+  ${PROJECT_SOURCE_DIR}/../test.py
+  ${PROJECT_NAME}
+  DEPENDS test_installed_static)

tests/test_cmake_build/subdirectory_static/CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.4)
+
+# The `cmake_minimum_required(VERSION 3.4...3.18)` syntax does not work with
+# some versions of VS that have a patched CMake 3.11. This forces us to emulate
+# the behavior using the following workaround:
+if(${CMAKE_VERSION} VERSION_LESS 3.18)
+  cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
+else()
+  cmake_policy(VERSION 3.18)
+endif()
+
+project(test_subdirectory_static CXX)
+
+add_subdirectory("${pybind11_SOURCE_DIR}" pybind11)
+
+add_library(test_subdirectory_static MODULE ../main.cpp)
+set_target_properties(test_subdirectory_static PROPERTIES OUTPUT_NAME test_cmake_build)
+
+target_link_libraries(test_subdirectory_static PRIVATE pybind11::module pybind11::static)
+
+# Make sure result is, for example, test_installed_static.so, not libtest_installed_static.dylib
+pybind11_extension(test_subdirectory_static)
+
+if(DEFINED Python_EXECUTABLE)
+  set(_Python_EXECUTABLE "${Python_EXECUTABLE}")
+elseif(DEFINED PYTHON_EXECUTABLE)
+  set(_Python_EXECUTABLE "${PYTHON_EXECUTABLE}")
+else()
+  message(FATAL_ERROR "No Python executable defined (should not be possible at this stage)")
+endif()
+
+add_custom_target(
+  check_subdirectory_static
+  ${CMAKE_COMMAND}
+  -E
+  env
+  PYTHONPATH=$<TARGET_FILE_DIR:test_subdirectory_static>
+  ${_Python_EXECUTABLE}
+  ${PROJECT_SOURCE_DIR}/../test.py
+  ${PROJECT_NAME}
+  DEPENDS test_subdirectory_static)

tools/pybind11Common.cmake

@@ -9,7 +9,8 @@ Adds the following targets::
     pybind11::thin_lto - Link time optimizations (manual selection)
     pybind11::python_link_helper - Adds link to Python libraries
     pybind11::windows_extras - MSVC bigobj and mp for building multithreaded
-    pybind11::opt_size - avoid optimizations that increase code size
+    pybind11::opt_size - Avoid optimizations that increase code size
+    pybind11::static - Use pybind11 as a static library.
 
 Adds the following functions::
 

tools/pybind11Config.cmake.in

@@ -49,6 +49,8 @@ complex applications, and they are available in all modes:
   Just the pybind11 headers and minimum compile requirements.
 ``pybind11::pybind11``
   Python headers too.
+``pybind11::static``
+  A static library to link against instead of the default header-only.
 ``pybind11::python_link_helper``
   Just the "linking" part of ``pybind11:module``, for CMake < 3.15.
 ``pybind11::thin_lto``
@@ -220,6 +222,11 @@ include("${CMAKE_CURRENT_LIST_DIR}/pybind11Targets.cmake")
 add_library(pybind11::headers IMPORTED INTERFACE)
 set_target_properties(pybind11::headers PROPERTIES INTERFACE_LINK_LIBRARIES
                                                    pybind11::pybind11_headers)
+if(@PYBIND11_BUILD_STATIC_LIB@)
+  add_library(pybind11::static IMPORTED INTERFACE)
+  set_target_properties(pybind11::static PROPERTIES INTERFACE_LINK_LIBRARIES
+                                                    pybind11::pybind11_static)
+endif()
 
 include("${CMAKE_CURRENT_LIST_DIR}/pybind11Common.cmake")