From 2a7dda26508e6dc7dd1222d9fe2dab100a261ee0 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 16:43:31 +0100 Subject: [PATCH 01/28] Added rework of arg/return typing --- include/pybind11/detail/descr.h | 7 +++++++ include/pybind11/pybind11.h | 23 +++++++++++++++++++++++ include/pybind11/stl/filesystem.h | 4 +--- tests/test_stl.py | 4 ++-- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 635614b0d6..e8bee4a191 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -99,6 +99,13 @@ constexpr descr<1, Type> const_name() { return {'%'}; } +// Use a different name based on whether the parameter is used as input or output +template +constexpr auto io_name(char const (&text1)[N1], char const (&text2)[N2]) { + return const_name("@") + const_name(text1) + const_name("@") + const_name(text2) + + const_name("@"); +} + // If "_" is defined as a macro, py::detail::_ cannot be provided. // It is therefore best to use py::detail::const_name universally. // This block is for backward compatibility only. diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 4387c2754b..fa1db2cce8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -440,6 +440,7 @@ class cpp_function : public function { std::string signature; size_t type_index = 0, arg_index = 0; bool is_starred = false; + bool is_return_value = false; for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; @@ -493,7 +494,29 @@ class cpp_function : public function { } else { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } + } else if (c == '@') { + // Handle types that differ depending on whether they appear + // in an argument or a return value position + ++pc; + if (!is_return_value) { + while (*pc && *pc != '@') + signature += *pc++; + if (*pc == '@') + ++pc; + while (*pc && *pc != '@') + ++pc; + } else { + while (*pc && *pc != '@') + ++pc; + if (*pc == '@') + ++pc; + while (*pc && *pc != '@') + signature += *pc++; + } } else { + if (c == '-' && *(pc + 1) == '>') { + is_return_value = true; + } signature += c; } } diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h index ecfb9cf0dc..46aa773ff5 100644 --- a/include/pybind11/stl/filesystem.h +++ b/include/pybind11/stl/filesystem.h @@ -106,9 +106,7 @@ struct path_caster { return true; } - PYBIND11_TYPE_CASTER(T, const_name("os.PathLike")); - static constexpr auto arg_name = const_name("Union[os.PathLike, str, bytes]"); - static constexpr auto return_name = const_name("Path"); + PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "Path")); }; #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) diff --git a/tests/test_stl.py b/tests/test_stl.py index 14c7da312a..ba997898b0 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -267,11 +267,11 @@ def __fspath__(self): doc(m.parent_path) == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> Path" ) - # std::vector should use name (for arg_name/return_name typing classes must be used) + # std::vector assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] assert ( doc(m.parent_paths) - == "parent_paths(arg0: list[os.PathLike]) -> list[os.PathLike]" + == "parent_paths(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" ) # py::typing::List assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] From 9d0766c61acee8b1dfc593ae6f1db2cffc8c9e92 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 16:50:33 +0100 Subject: [PATCH 02/28] Changed `Path` to `pathlib.Path` for compatibility with pybind11-stubgen --- include/pybind11/stl/filesystem.h | 2 +- tests/test_stl.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h index 46aa773ff5..fb8164e0d5 100644 --- a/include/pybind11/stl/filesystem.h +++ b/include/pybind11/stl/filesystem.h @@ -106,7 +106,7 @@ struct path_caster { return true; } - PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "Path")); + PYBIND11_TYPE_CASTER(T, io_name("Union[os.PathLike, str, bytes]", "pathlib.Path")); }; #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) diff --git a/tests/test_stl.py b/tests/test_stl.py index ba997898b0..f2ff727d97 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -265,19 +265,19 @@ def __fspath__(self): assert m.parent_path(PseudoBytesPath()) == Path("foo") assert ( doc(m.parent_path) - == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> Path" + == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> pathlib.Path" ) # std::vector assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] assert ( doc(m.parent_paths) - == "parent_paths(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" + == "parent_paths(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]" ) # py::typing::List assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] assert ( doc(m.parent_paths_list) - == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" + == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[pathlib.Path]" ) # Nested py::typing::List assert m.parent_paths_nested_list([["foo/bar"], ["foo/baz", "foo/buzz"]]) == [ @@ -286,13 +286,13 @@ def __fspath__(self): ] assert ( doc(m.parent_paths_nested_list) - == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[Path]]" + == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[pathlib.Path]]" ) # py::typing::Tuple assert m.parent_paths_tuple(("foo/bar", "foo/baz")) == (Path("foo"), Path("foo")) assert ( doc(m.parent_paths_tuple) - == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[Path, Path]" + == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[pathlib.Path, pathlib.Path]" ) # py::typing::Dict assert m.parent_paths_dict( @@ -308,7 +308,7 @@ def __fspath__(self): } assert ( doc(m.parent_paths_dict) - == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, Path]" + == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, pathlib.Path]" ) From d9f4e1ba9ed383bbf5ddda17fb8ad11a2c0d8ad2 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 17:25:15 +0100 Subject: [PATCH 03/28] Removed old arg/return type hint implementation --- docs/advanced/cast/custom.rst | 14 +++--- include/pybind11/cast.h | 37 +------------- include/pybind11/pybind11.h | 4 +- include/pybind11/typing.h | 63 +++--------------------- tests/test_docs_advanced_cast_custom.cpp | 14 +++--- tests/test_pytypes.cpp | 5 +- tests/test_pytypes.py | 22 +++++---- 7 files changed, 34 insertions(+), 125 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 065d09a6dd..5a626f3ba8 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -61,19 +61,16 @@ type is explicitly allowed. template <> struct type_caster { - // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple` - PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); - // `arg_name` and `return_name` may optionally be used to specify type hints separately for - // arguments and return values. + // This macro inserts a lot of boilerplate code and sets the type hint. + // `io_name` is used to specify different type hints for arguments and return values. // The signature of our negate function would then look like: // `negate(Sequence[float]) -> tuple[float, float]` - static constexpr auto arg_name = const_name("Sequence[float]"); - static constexpr auto return_name = const_name("tuple[float, float]"); + PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]")); // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. - // The return value should reflect the type hint specified by `return_name`. + // The return value should reflect the type hint specified by the second argument of `io_name`. static handle cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { return py::make_tuple(number.x, number.y).release(); @@ -81,7 +78,8 @@ type is explicitly allowed. // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The // second argument indicates whether implicit conversions should be allowed. - // The accepted types should reflect the type hint specified by `arg_name`. + // The accepted types should reflect the type hint specified by the first argument of + // `io_name`. bool load(handle src, bool /*convert*/) { // Check if handle is a Sequence if (!py::isinstance(src)) { diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 853165a498..f2c029113a 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -34,39 +34,6 @@ PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) -// Type trait checker for `descr` -template -struct is_descr : std::false_type {}; - -template -struct is_descr> : std::true_type {}; - -template -struct is_descr> : std::true_type {}; - -// Use arg_name instead of name when available -template -struct as_arg_type { - static constexpr auto name = T::name; -}; - -template -struct as_arg_type::value>::type> { - static constexpr auto name = T::arg_name; -}; - -// Use return_name instead of name when available -template -struct as_return_type { - static constexpr auto name = T::name; -}; - -template -struct as_return_type::value>::type> { - static constexpr auto name = T::return_name; -}; - template class type_caster : public type_caster_base {}; template @@ -1113,8 +1080,6 @@ struct pyobject_caster { return src.inc_ref(); } PYBIND11_TYPE_CASTER(type, handle_type_name::name); - static constexpr auto arg_name = as_arg_type>::name; - static constexpr auto return_name = as_return_type>::name; }; template @@ -1668,7 +1633,7 @@ class argument_loader { "py::args cannot be specified more than once"); static constexpr auto arg_names - = ::pybind11::detail::concat(type_descr(as_arg_type>::name)...); + = ::pybind11::detail::concat(type_descr(make_caster::name)...); bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index fa1db2cce8..64d0d15dfd 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -336,8 +336,8 @@ class cpp_function : public function { /* Generate a readable signature describing the function's arguments and return value types */ - static constexpr auto signature = const_name("(") + cast_in::arg_names - + const_name(") -> ") + as_return_type::name; + static constexpr auto signature + = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name; PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 005279058b..9693addb33 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -143,13 +143,6 @@ struct handle_type_name> { static constexpr auto name = const_name("tuple[") + ::pybind11::detail::concat(make_caster::name...) + const_name("]"); - static constexpr auto arg_name - = const_name("tuple[") - + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("]"); - static constexpr auto return_name - = const_name("tuple[") - + ::pybind11::detail::concat(as_return_type>::name...) - + const_name("]"); }; template <> @@ -163,76 +156,48 @@ struct handle_type_name> { // PEP 484 specifies this syntax for a variable-length tuple static constexpr auto name = const_name("tuple[") + make_caster::name + const_name(", ...]"); - static constexpr auto arg_name - = const_name("tuple[") + as_arg_type>::name + const_name(", ...]"); - static constexpr auto return_name - = const_name("tuple[") + as_return_type>::name + const_name(", ...]"); }; template struct handle_type_name> { static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") + make_caster::name + const_name("]"); - static constexpr auto arg_name = const_name("dict[") + as_arg_type>::name - + const_name(", ") + as_arg_type>::name - + const_name("]"); - static constexpr auto return_name = const_name("dict[") + as_return_type>::name - + const_name(", ") + as_return_type>::name - + const_name("]"); }; template struct handle_type_name> { static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("list[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("list[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("set[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("set[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { static constexpr auto name = const_name("Iterable[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("Iterable[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("Iterable[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { static constexpr auto name = const_name("Iterator[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("Iterator[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("Iterator[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { using retval_type = conditional_t::value, void_type, Return>; static constexpr auto name - = const_name("Callable[[") - + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("], ") - + as_return_type>::name + const_name("]"); + = const_name("Callable[[") + ::pybind11::detail::concat(make_caster::name...) + + const_name("], ") + make_caster::name + const_name("]"); }; template struct handle_type_name> { // PEP 484 specifies this syntax for defining only return types of callables using retval_type = conditional_t::value, void_type, Return>; - static constexpr auto name = const_name("Callable[..., ") - + as_return_type>::name - + const_name("]"); + static constexpr auto name + = const_name("Callable[..., ") + make_caster::name + const_name("]"); }; template @@ -245,22 +210,11 @@ struct handle_type_name> { static constexpr auto name = const_name("Union[") + ::pybind11::detail::concat(make_caster::name...) + const_name("]"); - static constexpr auto arg_name - = const_name("Union[") - + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("]"); - static constexpr auto return_name - = const_name("Union[") - + ::pybind11::detail::concat(as_return_type>::name...) - + const_name("]"); }; template struct handle_type_name> { static constexpr auto name = const_name("Optional[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("Optional[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("Optional[") + as_return_type>::name + const_name("]"); }; template @@ -273,19 +227,14 @@ struct handle_type_name> { static constexpr auto name = const_name("ClassVar[") + make_caster::name + const_name("]"); }; -// TypeGuard and TypeIs use as_return_type to use the return type if available, which is usually -// the narrower type. - template struct handle_type_name> { - static constexpr auto name - = const_name("TypeGuard[") + as_return_type>::name + const_name("]"); + static constexpr auto name = const_name("TypeGuard[") + make_caster::name + const_name("]"); }; template struct handle_type_name> { - static constexpr auto name - = const_name("TypeIs[") + as_return_type>::name + const_name("]"); + static constexpr auto name = const_name("TypeIs[") + make_caster::name + const_name("]"); }; template <> diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index a6f8a212ef..0ec1b17ac6 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -20,19 +20,16 @@ namespace detail { template <> struct type_caster { - // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple` - PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); - // `arg_name` and `return_name` may optionally be used to specify type hints separately for - // arguments and return values. + // This macro inserts a lot of boilerplate code and sets the type hint. + // `io_name` is used to specify different type hints for arguments and return values. // The signature of our negate function would then look like: // `negate(Sequence[float]) -> tuple[float, float]` - static constexpr auto arg_name = const_name("Sequence[float]"); - static constexpr auto return_name = const_name("tuple[float, float]"); + PYBIND11_TYPE_CASTER(user_space::Point2D, io_name("Sequence[float]", "tuple[float, float]")); // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. - // The return value should reflect the type hint specified by `return_name`. + // The return value should reflect the type hint specified by the second argument of `io_name`. static handle cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { return py::make_tuple(number.x, number.y).release(); @@ -40,7 +37,8 @@ struct type_caster { // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The // second argument indicates whether implicit conversions should be allowed. - // The accepted types should reflect the type hint specified by `arg_name`. + // The accepted types should reflect the type hint specified by the first argument of + // `io_name`. bool load(handle src, bool /*convert*/) { // Check if handle is a Sequence if (!py::isinstance(src)) { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index b4fa99192b..005573e453 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -142,7 +142,6 @@ typedef py::typing::TypeVar<"V"> TypeVarV; // RealNumber: // * in arguments -> float | int // * in return -> float -// * fallback -> complex // The choice of types is not really useful, but just made different for testing purposes. // According to `PEP 484 – Type Hints` annotating with `float` also allows `int`, // so using `float | int` could be replaced by just `float`. @@ -156,9 +155,7 @@ namespace detail { template <> struct type_caster { - PYBIND11_TYPE_CASTER(RealNumber, const_name("complex")); - static constexpr auto arg_name = const_name("Union[float, int]"); - static constexpr auto return_name = const_name("float"); + PYBIND11_TYPE_CASTER(RealNumber, io_name("Union[float, int]", "float")); static handle cast(const RealNumber &number, return_value_policy, handle) { return py::float_(number.value).release(); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 448bfa6a83..5651292e59 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1200,10 +1200,10 @@ def test_arg_return_type_hints(doc): assert m.half_of_number(0) == 0 assert isinstance(m.half_of_number(0), float) assert not isinstance(m.half_of_number(0), int) - # std::vector should use fallback type (complex is not really useful but just used for testing) + # std::vector assert ( doc(m.half_of_number_vector) - == "half_of_number_vector(arg0: list[complex]) -> list[complex]" + == "half_of_number_vector(arg0: list[Union[float, int]]) -> list[float]" ) # Tuple assert ( @@ -1246,15 +1246,17 @@ def test_arg_return_type_hints(doc): == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]" ) # Callable - assert ( - doc(m.apply_callable) - == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" - ) + # TODO: Needs support for arg/return environments + # assert ( + # doc(m.apply_callable) + # == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" + # ) # Callable - assert ( - doc(m.apply_callable_ellipsis) - == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" - ) + # TODO: Needs support for arg/return environments + # assert ( + # doc(m.apply_callable_ellipsis) + # == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" + # ) # Union assert ( doc(m.identity_union) From a86e7c47381cb434bc3f8fb6d437a33ac72a6331 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 18:55:12 +0100 Subject: [PATCH 04/28] Added noconvert support for arg/return type hints --- include/pybind11/pybind11.h | 4 +++- tests/test_pytypes.cpp | 14 +++++++++++++- tests/test_pytypes.py | 7 +++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 64d0d15dfd..b9ec5698f4 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -497,8 +497,10 @@ class cpp_function : public function { } else if (c == '@') { // Handle types that differ depending on whether they appear // in an argument or a return value position + // For named arguments (py::arg()) with noconvert set, use return value type ++pc; - if (!is_return_value) { + if (!is_return_value + && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) { while (*pc && *pc != '@') signature += *pc++; if (*pc == '@') diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 005573e453..16fafe0ba4 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -161,7 +161,11 @@ struct type_caster { return py::float_(number.value).release(); } - bool load(handle src, bool) { + bool load(handle src, bool convert) { + // If we're in no-convert mode, only load if given a float + if (!convert && !py::isinstance(src)) { + return false; + } if (!py::isinstance(src) && !py::isinstance(src)) { return false; } @@ -1067,6 +1071,14 @@ TEST_SUBMODULE(pytypes, m) { m.attr("defined___cpp_inline_variables") = false; #endif m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; }); + m.def( + "half_of_number_convert", + [](const RealNumber &x) { return RealNumber{x.value / 2}; }, + py::arg("x")); + m.def( + "half_of_number_noconvert", + [](const RealNumber &x) { return RealNumber{x.value / 2}; }, + py::arg("x").noconvert()); // std::vector m.def("half_of_number_vector", [](const std::vector &x) { std::vector result; diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 5651292e59..276f686095 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1195,6 +1195,13 @@ def test_final_annotation() -> None: def test_arg_return_type_hints(doc): assert doc(m.half_of_number) == "half_of_number(arg0: Union[float, int]) -> float" + assert ( + doc(m.half_of_number_convert) + == "half_of_number_convert(x: Union[float, int]) -> float" + ) + assert ( + doc(m.half_of_number_noconvert) == "half_of_number_noconvert(x: float) -> float" + ) assert m.half_of_number(2.0) == 1.0 assert m.half_of_number(2) == 1.0 assert m.half_of_number(0) == 0 From 9877aca5352d3875f6b5918c89816750573df611 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 21:26:54 +0100 Subject: [PATCH 05/28] Added commented failing tests for Literals with special characters --- tests/test_pytypes.cpp | 7 +++++++ tests/test_pytypes.py | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 16fafe0ba4..c212ea804c 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1168,4 +1168,11 @@ TEST_SUBMODULE(pytypes, m) { m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { return py::isinstance(x); }); + // Literal with `@`, `%`, `{`, and `}` + // m.def("identity_literal_x", [](const py::typing::Literal<"\"x\""> &x) { return x; }); + // m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); + // m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); + // m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; + // }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return + // x; }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 276f686095..1d999dfdc8 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1281,3 +1281,29 @@ def test_arg_return_type_hints(doc): ) # TypeIs assert doc(m.check_type_is) == "check_type_is(arg0: object) -> TypeIs[float]" + # Literal without special characters + # assert ( + # doc(m.identity_literal_x) + # == 'identity_literal_x(arg0: Literal["x"]) -> Literal["x"]' + # ) + # The Following tests fail with `ImportError: Internal error while parsing type signature (2)` + # Literal with @ + # assert ( + # doc(m.identity_literal_at) + # == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' + # ) + # Literal with % + # assert ( + # doc(m.identity_literal_percent) + # == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' + # ) + # Literal with { + # assert ( + # doc(m.identity_literal_curly_open) + # == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' + # ) + # Literal with } + # assert ( + # doc(m.identity_literal_curly_close) + # == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' + # ) From 077d0b6507f310bf9ec0c83c8066ef937e157181 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 5 Jan 2025 23:47:18 +0100 Subject: [PATCH 06/28] Added return_descr/arg_descr for correct typing in typing::Callable --- include/pybind11/detail/descr.h | 10 ++++++++++ include/pybind11/pybind11.h | 21 ++++++++++++++++++--- include/pybind11/typing.h | 11 +++++++---- tests/test_pytypes.cpp | 6 ++++++ tests/test_pytypes.py | 28 ++++++++++++++++++---------- 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index e8bee4a191..f45822079f 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -174,5 +174,15 @@ constexpr descr type_descr(const descr &descr) { return const_name("{") + descr + const_name("}"); } +template +constexpr descr arg_descr(const descr &descr) { + return const_name("@^") + descr + const_name("@^"); +} + +template +constexpr descr return_descr(const descr &descr) { + return const_name("@$") + descr + const_name("@$"); +} + PYBIND11_NAMESPACE_END(detail) PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index b9ec5698f4..9705cf3c93 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -440,7 +440,11 @@ class cpp_function : public function { std::string signature; size_t type_index = 0, arg_index = 0; bool is_starred = false; + // `is_return_value` is true if we are currently inside the return type of the signature. + // The same is true for `use_return_value`, except for forced usage of arg/return type + // using @^/@$. bool is_return_value = false; + bool use_return_value = false; for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; @@ -495,11 +499,21 @@ class cpp_function : public function { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } } else if (c == '@') { + // `@^ ... @^` and `@$ ... @$` are used to force arg/return value type (see + // typing::Callable/detail::arg_descr/detail::return_descr) + if ((*(pc + 1) == '^' && is_return_value) + || (*(pc + 1) == '$' && !is_return_value)) { + use_return_value = !use_return_value; + } + if (*(pc + 1) == '^' || *(pc + 1) == '$') { + ++pc; + continue; + } // Handle types that differ depending on whether they appear - // in an argument or a return value position - // For named arguments (py::arg()) with noconvert set, use return value type + // in an argument or a return value position (see io_name). + // For named arguments (py::arg()) with noconvert set, return value type is used. ++pc; - if (!is_return_value + if (!use_return_value && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) { while (*pc && *pc != '@') signature += *pc++; @@ -518,6 +532,7 @@ class cpp_function : public function { } else { if (c == '-' && *(pc + 1) == '>') { is_return_value = true; + use_return_value = true; } signature += c; } diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 9693addb33..fac145bfc5 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -188,16 +188,19 @@ template struct handle_type_name> { using retval_type = conditional_t::value, void_type, Return>; static constexpr auto name - = const_name("Callable[[") + ::pybind11::detail::concat(make_caster::name...) - + const_name("], ") + make_caster::name + const_name("]"); + = const_name("Callable[[") + + ::pybind11::detail::concat(::pybind11::detail::arg_descr(make_caster::name)...) + + const_name("], ") + ::pybind11::detail::return_descr(make_caster::name) + + const_name("]"); }; template struct handle_type_name> { // PEP 484 specifies this syntax for defining only return types of callables using retval_type = conditional_t::value, void_type, Return>; - static constexpr auto name - = const_name("Callable[..., ") + make_caster::name + const_name("]"); + static constexpr auto name = const_name("Callable[..., ") + + ::pybind11::detail::return_descr(make_caster::name) + + const_name("]"); }; template diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index c212ea804c..0221b59878 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1139,6 +1139,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("identity_iterable", [](const py::typing::Iterable &x) { return x; }); // Iterator m.def("identity_iterator", [](const py::typing::Iterator &x) { return x; }); + // Callable identity + m.def("identity_callable", + [](const py::typing::Callable &x) { return x; }); + // Callable identity + m.def("identity_callable_ellipsis", + [](const py::typing::Callable &x) { return x; }); // Callable m.def("apply_callable", [](const RealNumber &x, const py::typing::Callable &f) { diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 1d999dfdc8..f33b12d688 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1252,18 +1252,26 @@ def test_arg_return_type_hints(doc): doc(m.identity_iterator) == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]" ) + # Callable identity + assert ( + doc(m.identity_callable) + == "identity_callable(arg0: Callable[[Union[float, int]], float]) -> Callable[[Union[float, int]], float]" + ) + # Callable identity + assert ( + doc(m.identity_callable_ellipsis) + == "identity_callable_ellipsis(arg0: Callable[..., float]) -> Callable[..., float]" + ) # Callable - # TODO: Needs support for arg/return environments - # assert ( - # doc(m.apply_callable) - # == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" - # ) + assert ( + doc(m.apply_callable) + == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" + ) # Callable - # TODO: Needs support for arg/return environments - # assert ( - # doc(m.apply_callable_ellipsis) - # == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" - # ) + assert ( + doc(m.apply_callable_ellipsis) + == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" + ) # Union assert ( doc(m.identity_union) From 0a677e72f1dbbe91ca6d7a48a1d0f931e36860f8 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 09:57:51 +0100 Subject: [PATCH 07/28] Fixed clang-tidy issues --- include/pybind11/pybind11.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 9705cf3c93..68b6160013 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -515,19 +515,25 @@ class cpp_function : public function { ++pc; if (!use_return_value && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) { - while (*pc && *pc != '@') + while (*pc != '\0' && *pc != '@') { signature += *pc++; - if (*pc == '@') + } + if (*pc == '@') { ++pc; - while (*pc && *pc != '@') + } + while (*pc != '\0' && *pc != '@') { ++pc; + } } else { - while (*pc && *pc != '@') + while (*pc != '\0' && *pc != '@') { ++pc; - if (*pc == '@') + } + if (*pc == '@') { ++pc; - while (*pc && *pc != '@') + } + while (*pc != '\0' && *pc != '@') { signature += *pc++; + } } } else { if (c == '-' && *(pc + 1) == '>') { From 8b3bd83d53d8636875f544f909b7bfc99a4f48e7 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:03:03 +0100 Subject: [PATCH 08/28] Changed io_name to have explicit return type (for C++11 support) --- include/pybind11/detail/descr.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index f45822079f..ac608a64de 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -101,7 +101,7 @@ constexpr descr<1, Type> const_name() { // Use a different name based on whether the parameter is used as input or output template -constexpr auto io_name(char const (&text1)[N1], char const (&text2)[N2]) { +constexpr descr io_name(char const (&text1)[N1], char const (&text2)[N2]) { return const_name("@") + const_name(text1) + const_name("@") + const_name(text2) + const_name("@"); } @@ -163,8 +163,9 @@ constexpr auto concat(const descr &d, const Args &...args) { } #else template -constexpr auto concat(const descr &d, const Args &...args) - -> decltype(std::declval>() + concat(args...)) { +constexpr auto concat(const descr &d, + const Args &...args) -> decltype(std::declval>() + + concat(args...)) { return d + const_name(", ") + concat(args...); } #endif From 8718c509b1159203cb13e372c0254a2974d40987 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 09:03:46 +0000 Subject: [PATCH 09/28] style: pre-commit fixes --- include/pybind11/detail/descr.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index ac608a64de..603292df8b 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -163,9 +163,8 @@ constexpr auto concat(const descr &d, const Args &...args) { } #else template -constexpr auto concat(const descr &d, - const Args &...args) -> decltype(std::declval>() - + concat(args...)) { +constexpr auto concat(const descr &d, const Args &...args) + -> decltype(std::declval>() + concat(args...)) { return d + const_name(", ") + concat(args...); } #endif From 89de7a09444fe37b5b082cc3c126614217591edd Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:26:57 +0100 Subject: [PATCH 10/28] Added support for nested callables --- include/pybind11/detail/descr.h | 4 ++-- include/pybind11/pybind11.h | 30 +++++++++++++++++------------- tests/test_pytypes.cpp | 4 ++++ tests/test_pytypes.py | 5 +++++ 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/include/pybind11/detail/descr.h b/include/pybind11/detail/descr.h index 603292df8b..a5f17f8695 100644 --- a/include/pybind11/detail/descr.h +++ b/include/pybind11/detail/descr.h @@ -176,12 +176,12 @@ constexpr descr type_descr(const descr &descr) { template constexpr descr arg_descr(const descr &descr) { - return const_name("@^") + descr + const_name("@^"); + return const_name("@^") + descr + const_name("@!"); } template constexpr descr return_descr(const descr &descr) { - return const_name("@$") + descr + const_name("@$"); + return const_name("@$") + descr + const_name("@!"); } PYBIND11_NAMESPACE_END(detail) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 68b6160013..372f6e1b52 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -440,10 +440,10 @@ class cpp_function : public function { std::string signature; size_t type_index = 0, arg_index = 0; bool is_starred = false; - // `is_return_value` is true if we are currently inside the return type of the signature. - // The same is true for `use_return_value`, except for forced usage of arg/return type - // using @^/@$. - bool is_return_value = false; + // `is_return_value.top()` is true if we are currently inside the return type of the + // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops + // back to the previous state. + std::stack is_return_value = {false}; bool use_return_value = false; for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; @@ -499,13 +499,18 @@ class cpp_function : public function { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } } else if (c == '@') { - // `@^ ... @^` and `@$ ... @$` are used to force arg/return value type (see + // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see // typing::Callable/detail::arg_descr/detail::return_descr) - if ((*(pc + 1) == '^' && is_return_value) - || (*(pc + 1) == '$' && !is_return_value)) { - use_return_value = !use_return_value; - } - if (*(pc + 1) == '^' || *(pc + 1) == '$') { + if (*(pc + 1) == '^') { + is_return_value.emplace(false); + ++pc; + continue; + } else if (*(pc + 1) == '$') { + is_return_value.emplace(true); + ++pc; + continue; + } else if (*(pc + 1) == '!') { + is_return_value.pop(); ++pc; continue; } @@ -513,7 +518,7 @@ class cpp_function : public function { // in an argument or a return value position (see io_name). // For named arguments (py::arg()) with noconvert set, return value type is used. ++pc; - if (!use_return_value + if (!is_return_value.top() && !(arg_index < rec->args.size() && !rec->args[arg_index].convert)) { while (*pc != '\0' && *pc != '@') { signature += *pc++; @@ -537,8 +542,7 @@ class cpp_function : public function { } } else { if (c == '-' && *(pc + 1) == '>') { - is_return_value = true; - use_return_value = true; + is_return_value.emplace(true); } signature += c; } diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 0221b59878..935528c468 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1145,6 +1145,10 @@ TEST_SUBMODULE(pytypes, m) { // Callable identity m.def("identity_callable_ellipsis", [](const py::typing::Callable &x) { return x; }); + // Nested Callable identity + m.def("identity_nested_callable", + [](const py::typing::Callable( + py::typing::Callable)> &x) { return x; }); // Callable m.def("apply_callable", [](const RealNumber &x, const py::typing::Callable &f) { diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index f33b12d688..339d39384b 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1262,6 +1262,11 @@ def test_arg_return_type_hints(doc): doc(m.identity_callable_ellipsis) == "identity_callable_ellipsis(arg0: Callable[..., float]) -> Callable[..., float]" ) + # Nested Callable identity + assert ( + doc(m.identity_nested_callable) + == "identity_nested_callable(arg0: Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]) -> Callable[[Callable[[Union[float, int]], float]], Callable[[Union[float, int]], float]]" + ) # Callable assert ( doc(m.apply_callable) From 2ba568cf16b44e4158ed7037a6913ce9439326b0 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:31:02 +0100 Subject: [PATCH 11/28] Fixed missing include --- include/pybind11/pybind11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 372f6e1b52..0f3d60a83d 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -444,7 +445,6 @@ class cpp_function : public function { // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops // back to the previous state. std::stack is_return_value = {false}; - bool use_return_value = false; for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; From e37cd4047600bb0f8c47a9b218f773c6411269be Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:36:38 +0100 Subject: [PATCH 12/28] Fixed is_return_value constructor call --- include/pybind11/pybind11.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 0f3d60a83d..a2140256ef 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -444,7 +444,7 @@ class cpp_function : public function { // `is_return_value.top()` is true if we are currently inside the return type of the // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops // back to the previous state. - std::stack is_return_value = {false}; + std::stack is_return_value({false}); for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; From 1dd503690dd518a287cd40be205a276ca3585400 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:49:10 +0100 Subject: [PATCH 13/28] Fixed clang-tidy issue --- include/pybind11/pybind11.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index a2140256ef..33c0f584c8 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -505,11 +505,13 @@ class cpp_function : public function { is_return_value.emplace(false); ++pc; continue; - } else if (*(pc + 1) == '$') { + } + if (*(pc + 1) == '$') { is_return_value.emplace(true); ++pc; continue; - } else if (*(pc + 1) == '!') { + } + if (*(pc + 1) == '!') { is_return_value.pop(); ++pc; continue; From 19678a0241a703c6bb8f029c79e7cd1c450db665 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:51:11 +0100 Subject: [PATCH 14/28] Uncommented test cases for special characters in literals --- tests/test_pytypes.cpp | 11 +++++------ tests/test_pytypes.py | 41 ++++++++++++++++++++--------------------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 935528c468..27761397ad 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1179,10 +1179,9 @@ TEST_SUBMODULE(pytypes, m) { return py::isinstance(x); }); // Literal with `@`, `%`, `{`, and `}` - // m.def("identity_literal_x", [](const py::typing::Literal<"\"x\""> &x) { return x; }); - // m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); - // m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); - // m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; - // }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return - // x; }); + m.def("identity_literal_x", [](const py::typing::Literal<"\"x\""> &x) { return x; }); + m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); + m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); + m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); + m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 339d39384b..f174045e66 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1295,28 +1295,27 @@ def test_arg_return_type_hints(doc): # TypeIs assert doc(m.check_type_is) == "check_type_is(arg0: object) -> TypeIs[float]" # Literal without special characters - # assert ( - # doc(m.identity_literal_x) - # == 'identity_literal_x(arg0: Literal["x"]) -> Literal["x"]' - # ) - # The Following tests fail with `ImportError: Internal error while parsing type signature (2)` + assert ( + doc(m.identity_literal_x) + == 'identity_literal_x(arg0: Literal["x"]) -> Literal["x"]' + ) # Literal with @ - # assert ( - # doc(m.identity_literal_at) - # == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' - # ) + assert ( + doc(m.identity_literal_at) + == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' + ) # Literal with % - # assert ( - # doc(m.identity_literal_percent) - # == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' - # ) + assert ( + doc(m.identity_literal_percent) + == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' + ) # Literal with { - # assert ( - # doc(m.identity_literal_curly_open) - # == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' - # ) + assert ( + doc(m.identity_literal_curly_open) + == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' + ) # Literal with } - # assert ( - # doc(m.identity_literal_curly_close) - # == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' - # ) + assert ( + doc(m.identity_literal_curly_close) + == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' + ) From 1b8873b10bd98fb2d1f3a3b04306b6ed5285a887 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 10:57:34 +0100 Subject: [PATCH 15/28] Moved literal tests to correct test case --- tests/test_pytypes.cpp | 12 +++++------- tests/test_pytypes.py | 42 +++++++++++++++++------------------------- 2 files changed, 22 insertions(+), 32 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 27761397ad..9b2fa95e19 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -971,6 +971,11 @@ TEST_SUBMODULE(pytypes, m) { .value("BLUE", literals::Color::BLUE); m.def("annotate_literal", [](literals::LiteralFoo &o) -> py::object { return o; }); + // Literal with `@`, `%`, `{`, and `}` + m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); + m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); + m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); + m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); m.def("annotate_generic_containers", [](const py::typing::List &l) -> py::typing::List { return l; @@ -1178,10 +1183,3 @@ TEST_SUBMODULE(pytypes, m) { m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { return py::isinstance(x); }); - // Literal with `@`, `%`, `{`, and `}` - m.def("identity_literal_x", [](const py::typing::Literal<"\"x\""> &x) { return x; }); - m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); - m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); - m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); - m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); -} diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index f174045e66..eb12d8411d 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1044,6 +1044,23 @@ def test_literal(doc): doc(m.annotate_literal) == 'annotate_literal(arg0: Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]) -> object' ) + # The characters @, %, and {} are used in the signature parser as special characters, but Literal should escape those for the parser to work. + assert ( + doc(m.identity_literal_at) + == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' + ) + assert ( + doc(m.identity_literal_percent) + == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' + ) + assert ( + doc(m.identity_literal_curly_open) + == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' + ) + assert ( + doc(m.identity_literal_curly_close) + == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' + ) @pytest.mark.skipif( @@ -1294,28 +1311,3 @@ def test_arg_return_type_hints(doc): ) # TypeIs assert doc(m.check_type_is) == "check_type_is(arg0: object) -> TypeIs[float]" - # Literal without special characters - assert ( - doc(m.identity_literal_x) - == 'identity_literal_x(arg0: Literal["x"]) -> Literal["x"]' - ) - # Literal with @ - assert ( - doc(m.identity_literal_at) - == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' - ) - # Literal with % - assert ( - doc(m.identity_literal_percent) - == 'identity_literal_percent(arg0: Literal["%"]) -> Literal["%"]' - ) - # Literal with { - assert ( - doc(m.identity_literal_curly_open) - == 'identity_literal_curly_open(arg0: Literal["{"]) -> Literal["{"]' - ) - # Literal with } - assert ( - doc(m.identity_literal_curly_close) - == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' - ) From e0e222591a9a7e65476d9939854c3eab3f37dd4b Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 13:45:46 +0100 Subject: [PATCH 16/28] Added escaping of special characters in typing::Literal --- include/pybind11/pybind11.h | 5 +++++ include/pybind11/typing.h | 39 +++++++++++++++++++++++++++++++++---- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 33c0f584c8..41ca4789ee 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -498,6 +498,11 @@ class cpp_function : public function { } else { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } + } else if (c == '!' + && (*(pc + 1) == '!' || *(pc + 1) == '@' || *(pc + 1) == '%' + || *(pc + 1) == '{' || *(pc + 1) == '}')) { + // typing::Literal escapes special characters with ! + signature += *++pc; } else if (c == '@') { // `@^ ... @!` and `@$ ... @!` are used to force arg/return value type (see // typing::Callable/detail::arg_descr/detail::return_descr) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index fac145bfc5..dade12b75c 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -120,6 +120,36 @@ struct StringLiteral { char name[N]; }; +template +consteval auto sanitize_string_literal() { + constexpr std::string_view v(str.name); + char result[v.size() + std::ranges::count(v, '!') + std::ranges::count(v, '@') + + std::ranges::count(v, '%') + std::ranges::count(v, '{') + + std::ranges::count(v, '}') + 1]; + size_t i = 0; + for (auto c : str.name) { + if (c == '!') { + result[i++] = '!'; + result[i++] = '!'; + } else if (c == '@') { + result[i++] = '!'; + result[i++] = '@'; + } else if (c == '%') { + result[i++] = '!'; + result[i++] = '%'; + } else if (c == '{') { + result[i++] = '!'; + result[i++] = '{'; + } else if (c == '}') { + result[i++] = '!'; + result[i++] = '}'; + } else { + result[i++] = c; + } + } + return StringLiteral(result); +} + template class Literal : public object { PYBIND11_OBJECT_DEFAULT(Literal, object, PyObject_Type) @@ -253,13 +283,14 @@ struct handle_type_name { #if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) template struct handle_type_name> { - static constexpr auto name = const_name("Literal[") - + pybind11::detail::concat(const_name(Literals.name)...) - + const_name("]"); + static constexpr auto name + = const_name("Literal[") + + pybind11::detail::concat(const_name(sanitize_string_literal().name)...) + + const_name("]"); }; template struct handle_type_name> { - static constexpr auto name = const_name(StrLit.name); + static constexpr auto name = const_name(sanitize_string_literal().name); }; #endif From b65e44b0d8755743df13cf44700e5a141cdc2904 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 14:04:50 +0100 Subject: [PATCH 17/28] Readded mistakenly deleted bracket --- tests/test_pytypes.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 9b2fa95e19..fd31bc854f 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1183,3 +1183,4 @@ TEST_SUBMODULE(pytypes, m) { m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { return py::isinstance(x); }); +} From 8df42eb616567eddfad0cf1c5975094f8d4ace50 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 14:23:03 +0100 Subject: [PATCH 18/28] Moved sanitize_string_literal to correct namespace --- include/pybind11/typing.h | 69 +++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index dade12b75c..f67cc84076 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -16,6 +16,12 @@ #include +#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L +# define PYBIND11_TYPING_H_HAS_STRING_LITERAL +# include +# include +#endif + PYBIND11_NAMESPACE_BEGIN(PYBIND11_NAMESPACE) PYBIND11_NAMESPACE_BEGIN(typing) @@ -112,44 +118,13 @@ class Never : public none { using none::none; }; -#if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L -# define PYBIND11_TYPING_H_HAS_STRING_LITERAL +#if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) template struct StringLiteral { constexpr StringLiteral(const char (&str)[N]) { std::copy_n(str, N, name); } char name[N]; }; -template -consteval auto sanitize_string_literal() { - constexpr std::string_view v(str.name); - char result[v.size() + std::ranges::count(v, '!') + std::ranges::count(v, '@') - + std::ranges::count(v, '%') + std::ranges::count(v, '{') - + std::ranges::count(v, '}') + 1]; - size_t i = 0; - for (auto c : str.name) { - if (c == '!') { - result[i++] = '!'; - result[i++] = '!'; - } else if (c == '@') { - result[i++] = '!'; - result[i++] = '@'; - } else if (c == '%') { - result[i++] = '!'; - result[i++] = '%'; - } else if (c == '{') { - result[i++] = '!'; - result[i++] = '{'; - } else if (c == '}') { - result[i++] = '!'; - result[i++] = '}'; - } else { - result[i++] = c; - } - } - return StringLiteral(result); -} - template class Literal : public object { PYBIND11_OBJECT_DEFAULT(Literal, object, PyObject_Type) @@ -281,6 +256,36 @@ struct handle_type_name { }; #if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) +template +consteval auto sanitize_string_literal() { + constexpr std::string_view v(str.name); + char result[v.size() + std::ranges::count(v, '!') + std::ranges::count(v, '@') + + std::ranges::count(v, '%') + std::ranges::count(v, '{') + + std::ranges::count(v, '}') + 1]; + size_t i = 0; + for (auto c : str.name) { + if (c == '!') { + result[i++] = '!'; + result[i++] = '!'; + } else if (c == '@') { + result[i++] = '!'; + result[i++] = '@'; + } else if (c == '%') { + result[i++] = '!'; + result[i++] = '%'; + } else if (c == '{') { + result[i++] = '!'; + result[i++] = '{'; + } else if (c == '}') { + result[i++] = '!'; + result[i++] = '}'; + } else { + result[i++] = c; + } + } + return typing::StringLiteral(result); +} + template struct handle_type_name> { static constexpr auto name From 98678000ee2e7153ed709538d8e54005b07343e6 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 14:36:24 +0100 Subject: [PATCH 19/28] Added test for Literal with `!` and changed StringLiteral template param name --- include/pybind11/typing.h | 6 +++--- tests/test_pytypes.cpp | 1 + tests/test_pytypes.py | 6 +++++- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index f67cc84076..88bac60d31 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -256,14 +256,14 @@ struct handle_type_name { }; #if defined(PYBIND11_TYPING_H_HAS_STRING_LITERAL) -template +template consteval auto sanitize_string_literal() { - constexpr std::string_view v(str.name); + constexpr std::string_view v(StrLit.name); char result[v.size() + std::ranges::count(v, '!') + std::ranges::count(v, '@') + std::ranges::count(v, '%') + std::ranges::count(v, '{') + std::ranges::count(v, '}') + 1]; size_t i = 0; - for (auto c : str.name) { + for (auto c : StrLit.name) { if (c == '!') { result[i++] = '!'; result[i++] = '!'; diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index fd31bc854f..fb39300e5c 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -972,6 +972,7 @@ TEST_SUBMODULE(pytypes, m) { m.def("annotate_literal", [](literals::LiteralFoo &o) -> py::object { return o; }); // Literal with `@`, `%`, `{`, and `}` + m.def("identity_literal_exclamation", [](const py::typing::Literal<"\"!\""> &x) { return x; }); m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index eb12d8411d..10f04f6790 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1044,7 +1044,11 @@ def test_literal(doc): doc(m.annotate_literal) == 'annotate_literal(arg0: Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]) -> object' ) - # The characters @, %, and {} are used in the signature parser as special characters, but Literal should escape those for the parser to work. + # The characters !, @, %, and {} are used in the signature parser as special characters, but Literal should escape those for the parser to work. + assert ( + doc(m.identity_literal_exclamation) + == 'identity_literal_exclamation(arg0: Literal["!"]) -> Literal["!"]' + ) assert ( doc(m.identity_literal_at) == 'identity_literal_at(arg0: Literal["@"]) -> Literal["@"]' From 23d9551f19d12c21f37202eb27d484ba32ad0aa6 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 9 Jan 2025 15:22:31 +0100 Subject: [PATCH 20/28] Added test for Literal with multiple and repeated special chars --- tests/test_pytypes.cpp | 2 ++ tests/test_pytypes.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index fb39300e5c..55d1101c61 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -977,6 +977,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); + m.def("identity_literal_all_special_chars", + [](const py::typing::Literal<"\"!@!!{%}\""> &x) { return x; }); m.def("annotate_generic_containers", [](const py::typing::List &l) -> py::typing::List { return l; diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 10f04f6790..e40a3fe393 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1065,6 +1065,10 @@ def test_literal(doc): doc(m.identity_literal_curly_close) == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' ) + assert ( + doc(m.identity_literal_all_special_chars) + == 'identity_literal_all_special_chars(arg0: Literal["!@!!{%}"]) -> Literal["!@!!{%}"]' + ) @pytest.mark.skipif( From b577012e5f96b7e608264116cc1b8993d4fe8a3c Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 09:16:57 +0100 Subject: [PATCH 21/28] Simplified string literal sanitization function --- include/pybind11/typing.h | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 88bac60d31..1028a5d572 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -264,24 +264,10 @@ consteval auto sanitize_string_literal() { + std::ranges::count(v, '}') + 1]; size_t i = 0; for (auto c : StrLit.name) { - if (c == '!') { + if (c == '!' || c == '@' || c == '%' || c == '{' || c == '}') { result[i++] = '!'; - result[i++] = '!'; - } else if (c == '@') { - result[i++] = '!'; - result[i++] = '@'; - } else if (c == '%') { - result[i++] = '!'; - result[i++] = '%'; - } else if (c == '{') { - result[i++] = '!'; - result[i++] = '{'; - } else if (c == '}') { - result[i++] = '!'; - result[i++] = '}'; - } else { - result[i++] = c; } + result[i++] = c; } return typing::StringLiteral(result); } From 9bb4f20e792adfff67b36ee4048e978154942779 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 09:28:33 +0100 Subject: [PATCH 22/28] Added test for `->` in literal --- tests/test_pytypes.cpp | 6 +++++- tests/test_pytypes.py | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 55d1101c61..e1e3347181 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -977,8 +977,12 @@ TEST_SUBMODULE(pytypes, m) { m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); + m.def("identity_literal_arrow", [](const py::typing::Literal<"\"->\""> &x) { return x; }); + m.def("identity_literal_arrow_with_callable", + [](const py::typing::Callable\""> &, + const RealNumber &)> &x) { return x; }); m.def("identity_literal_all_special_chars", - [](const py::typing::Literal<"\"!@!!{%}\""> &x) { return x; }); + [](const py::typing::Literal<"\"!@!!->{%}\""> &x) { return x; }); m.def("annotate_generic_containers", [](const py::typing::List &l) -> py::typing::List { return l; diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index e40a3fe393..a6f3ca6585 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1065,9 +1065,17 @@ def test_literal(doc): doc(m.identity_literal_curly_close) == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' ) + assert ( + doc(m.identity_literal_arrow) + == 'identity_literal_arrow(arg0: Literal["->"]) -> Literal["->"]' + ) + assert ( + doc(m.identity_literal_arrow_with_callable) + == 'identity_literal_arrow_with_callable(arg0: Callable[[Literal["->"], Union[float, int]], float]) -> Callable[[Literal["->"], Union[float, int]], float]' + ) assert ( doc(m.identity_literal_all_special_chars) - == 'identity_literal_all_special_chars(arg0: Literal["!@!!{%}"]) -> Literal["!@!!{%}"]' + == 'identity_literal_all_special_chars(arg0: Literal["!@!!->{%}"]) -> Literal["!@!!->{%}"]' ) From e97433c2b2cb36e6d75779804db439da4330863d Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 09:46:38 +0100 Subject: [PATCH 23/28] Added test for `->` with io_name --- tests/test_pytypes.cpp | 3 ++- tests/test_pytypes.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index e1e3347181..43a0247208 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -977,7 +977,8 @@ TEST_SUBMODULE(pytypes, m) { m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); - m.def("identity_literal_arrow", [](const py::typing::Literal<"\"->\""> &x) { return x; }); + m.def("identity_literal_arrow_with_io_name", + [](const py::typing::Literal<"\"->\""> &x, const RealNumber &y) { return x; }); m.def("identity_literal_arrow_with_callable", [](const py::typing::Callable\""> &, const RealNumber &)> &x) { return x; }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index a6f3ca6585..9edc6ea5ec 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1066,8 +1066,8 @@ def test_literal(doc): == 'identity_literal_curly_close(arg0: Literal["}"]) -> Literal["}"]' ) assert ( - doc(m.identity_literal_arrow) - == 'identity_literal_arrow(arg0: Literal["->"]) -> Literal["->"]' + doc(m.identity_literal_arrow_with_io_name) + == 'identity_literal_arrow_with_io_name(arg0: Literal["->"], arg1: Union[float, int]) -> Literal["->"]' ) assert ( doc(m.identity_literal_arrow_with_callable) From 56cab0b761a20f210cc3ae042b4317003bf42bb4 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 09:52:38 +0100 Subject: [PATCH 24/28] Removed unused parameter name to prevent warning --- tests/test_pytypes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 43a0247208..209513db9d 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -978,7 +978,7 @@ TEST_SUBMODULE(pytypes, m) { m.def("identity_literal_curly_open", [](const py::typing::Literal<"\"{\""> &x) { return x; }); m.def("identity_literal_curly_close", [](const py::typing::Literal<"\"}\""> &x) { return x; }); m.def("identity_literal_arrow_with_io_name", - [](const py::typing::Literal<"\"->\""> &x, const RealNumber &y) { return x; }); + [](const py::typing::Literal<"\"->\""> &x, const RealNumber &) { return x; }); m.def("identity_literal_arrow_with_callable", [](const py::typing::Callable\""> &, const RealNumber &)> &x) { return x; }); From 2029854b03b0d2974736f778689c60b255c978dd Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 10:00:25 +0100 Subject: [PATCH 25/28] Added escaping of `-` in literal to prevent processing of `->` --- include/pybind11/pybind11.h | 2 +- include/pybind11/typing.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 41ca4789ee..5e277d5a1a 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -500,7 +500,7 @@ class cpp_function : public function { } } else if (c == '!' && (*(pc + 1) == '!' || *(pc + 1) == '@' || *(pc + 1) == '%' - || *(pc + 1) == '{' || *(pc + 1) == '}')) { + || *(pc + 1) == '{' || *(pc + 1) == '}' || *(pc + 1) == '-')) { // typing::Literal escapes special characters with ! signature += *++pc; } else if (c == '@') { diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 1028a5d572..b65207fcca 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -264,7 +264,7 @@ consteval auto sanitize_string_literal() { + std::ranges::count(v, '}') + 1]; size_t i = 0; for (auto c : StrLit.name) { - if (c == '!' || c == '@' || c == '%' || c == '{' || c == '}') { + if (c == '!' || c == '@' || c == '%' || c == '{' || c == '}' || c == '-') { result[i++] = '!'; } result[i++] = c; From 4e49c81474d0723e242c44451774e9f4fc22deba Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 11:15:04 +0100 Subject: [PATCH 26/28] Fixed wrong computation of sanitized string literal length --- include/pybind11/typing.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index b65207fcca..6660855bbc 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -18,6 +18,7 @@ #if defined(__cpp_nontype_template_args) && __cpp_nontype_template_args >= 201911L # define PYBIND11_TYPING_H_HAS_STRING_LITERAL +# include # include # include #endif @@ -259,12 +260,15 @@ struct handle_type_name { template consteval auto sanitize_string_literal() { constexpr std::string_view v(StrLit.name); - char result[v.size() + std::ranges::count(v, '!') + std::ranges::count(v, '@') - + std::ranges::count(v, '%') + std::ranges::count(v, '{') - + std::ranges::count(v, '}') + 1]; + constexpr std::string_view special_chars("!@%{}-"); + constexpr auto num_special_chars = std::accumulate( + special_chars.begin(), special_chars.end(), 0, [&v](auto acc, const char &c) { + return std::move(acc) + std::ranges::count(v, c); + }); + char result[v.size() + num_special_chars + 1]; size_t i = 0; for (auto c : StrLit.name) { - if (c == '!' || c == '@' || c == '%' || c == '{' || c == '}' || c == '-') { + if (special_chars.find(c) != std::string_view::npos) { result[i++] = '!'; } result[i++] = c; From bbb0ca7b53873b1171a371f2b3095fa39fcd282e Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 11:30:45 +0100 Subject: [PATCH 27/28] Added cast to prevent error with MSVC --- include/pybind11/typing.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 6660855bbc..c5f342aed0 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -262,7 +262,7 @@ consteval auto sanitize_string_literal() { constexpr std::string_view v(StrLit.name); constexpr std::string_view special_chars("!@%{}-"); constexpr auto num_special_chars = std::accumulate( - special_chars.begin(), special_chars.end(), 0, [&v](auto acc, const char &c) { + special_chars.begin(), special_chars.end(), (size_t) 0, [&v](auto acc, const char &c) { return std::move(acc) + std::ranges::count(v, c); }); char result[v.size() + num_special_chars + 1]; From 21f52eb0c7778058ffe0c73317b1c1d5cecf7522 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Fri, 10 Jan 2025 11:59:37 +0100 Subject: [PATCH 28/28] Simplified special character check --- include/pybind11/pybind11.h | 7 ++++--- tests/test_pytypes.cpp | 2 +- tests/test_pytypes.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index 5e277d5a1a..655bec3986 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -445,6 +445,9 @@ class cpp_function : public function { // signature. Using `@^`/`@$` we can force types to be arg/return types while `@!` pops // back to the previous state. std::stack is_return_value({false}); + // The following characters have special meaning in the signature parsing. Literals + // containing these are escaped with `!`. + std::string special_chars("!@%{}-"); for (const auto *pc = text; *pc != '\0'; ++pc) { const auto c = *pc; @@ -498,9 +501,7 @@ class cpp_function : public function { } else { signature += detail::quote_cpp_type_name(detail::clean_type_id(t->name())); } - } else if (c == '!' - && (*(pc + 1) == '!' || *(pc + 1) == '@' || *(pc + 1) == '%' - || *(pc + 1) == '{' || *(pc + 1) == '}' || *(pc + 1) == '-')) { + } else if (c == '!' && special_chars.find(*(pc + 1)) != std::string::npos) { // typing::Literal escapes special characters with ! signature += *++pc; } else if (c == '@') { diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 209513db9d..5160e9f408 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -971,7 +971,7 @@ TEST_SUBMODULE(pytypes, m) { .value("BLUE", literals::Color::BLUE); m.def("annotate_literal", [](literals::LiteralFoo &o) -> py::object { return o; }); - // Literal with `@`, `%`, `{`, and `}` + // Literal with `@`, `%`, `{`, `}`, and `->` m.def("identity_literal_exclamation", [](const py::typing::Literal<"\"!\""> &x) { return x; }); m.def("identity_literal_at", [](const py::typing::Literal<"\"@\""> &x) { return x; }); m.def("identity_literal_percent", [](const py::typing::Literal<"\"%\""> &x) { return x; }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 9edc6ea5ec..469137cc3c 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1044,7 +1044,7 @@ def test_literal(doc): doc(m.annotate_literal) == 'annotate_literal(arg0: Literal[26, 0x1A, "hello world", b"hello world", u"hello world", True, Color.RED, None]) -> object' ) - # The characters !, @, %, and {} are used in the signature parser as special characters, but Literal should escape those for the parser to work. + # The characters !, @, %, {, } and -> are used in the signature parser as special characters, but Literal should escape those for the parser to work. assert ( doc(m.identity_literal_exclamation) == 'identity_literal_exclamation(arg0: Literal["!"]) -> Literal["!"]'