From 2b8de829128f5ec065039f9a99a9ed534d2f3716 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 13:53:48 +0100 Subject: [PATCH 01/32] Added arg/return type handling. --- include/pybind11/cast.h | 35 ++++++++++++++++++++++++++++++++++- include/pybind11/pybind11.h | 4 ++-- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 2ae25c2ebf..be49031ec1 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -1582,6 +1582,39 @@ template struct is_same_or_base_of : any_of, std::is_base_of> {}; +// Type trait checker for `descr` +template +struct is_descr : std::false_type {}; + +template +struct is_descr> : std::true_type {}; + +template +struct is_descr> : std::true_type {}; + +// Use arg_name instead of name when available +template +struct as_arg_type { + static constexpr auto name = T::name; +}; + +template +struct as_arg_type::value>::type> { + static constexpr auto name = T::arg_name; +}; + +// Use return_name instead of name when available +template +struct as_return_type { + static constexpr auto name = T::name; +}; + +template +struct as_return_type::value>::type> { + static constexpr auto name = T::return_name; +}; + /// Helper class which loads arguments for C++ functions called from Python template class argument_loader { @@ -1606,7 +1639,7 @@ class argument_loader { "py::args cannot be specified more than once"); static constexpr auto arg_names - = ::pybind11::detail::concat(type_descr(make_caster::name)...); + = ::pybind11::detail::concat(type_descr(as_arg_type>::name)...); bool load_args(function_call &call) { return load_impl_sequence(call, indices{}); } diff --git a/include/pybind11/pybind11.h b/include/pybind11/pybind11.h index fe8384e57f..ebb13bcc2b 100644 --- a/include/pybind11/pybind11.h +++ b/include/pybind11/pybind11.h @@ -330,8 +330,8 @@ class cpp_function : public function { /* Generate a readable signature describing the function's arguments and return value types */ - static constexpr auto signature - = const_name("(") + cast_in::arg_names + const_name(") -> ") + cast_out::name; + static constexpr auto signature = const_name("(") + cast_in::arg_names + + const_name(") -> ") + as_return_type::name; PYBIND11_DESCR_CONSTEXPR auto types = decltype(signature)::types(); /* Register the function with Python from generic (non-templated) code */ From 7ba983b14c8265feccc4d6c3704829bf07232407 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 17:24:49 +0100 Subject: [PATCH 02/32] Added support for nested arg/return type in py::typing::List --- include/pybind11/cast.h | 68 ++++++++++++++++++++------------------- include/pybind11/typing.h | 4 +++ 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index be49031ec1..3088a518a8 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -34,6 +34,39 @@ PYBIND11_WARNING_DISABLE_MSVC(4127) PYBIND11_NAMESPACE_BEGIN(detail) +// Type trait checker for `descr` +template +struct is_descr : std::false_type {}; + +template +struct is_descr> : std::true_type {}; + +template +struct is_descr> : std::true_type {}; + +// Use arg_name instead of name when available +template +struct as_arg_type { + static constexpr auto name = T::name; +}; + +template +struct as_arg_type::value>::type> { + static constexpr auto name = T::arg_name; +}; + +// Use return_name instead of name when available +template +struct as_return_type { + static constexpr auto name = T::name; +}; + +template +struct as_return_type::value>::type> { + static constexpr auto name = T::return_name; +}; + template class type_caster : public type_caster_base {}; template @@ -1078,6 +1111,8 @@ struct pyobject_caster { return src.inc_ref(); } PYBIND11_TYPE_CASTER(type, handle_type_name::name); + static constexpr auto arg_name = as_arg_type>::name; + static constexpr auto return_name = as_return_type>::name; }; template @@ -1582,39 +1617,6 @@ template struct is_same_or_base_of : any_of, std::is_base_of> {}; -// Type trait checker for `descr` -template -struct is_descr : std::false_type {}; - -template -struct is_descr> : std::true_type {}; - -template -struct is_descr> : std::true_type {}; - -// Use arg_name instead of name when available -template -struct as_arg_type { - static constexpr auto name = T::name; -}; - -template -struct as_arg_type::value>::type> { - static constexpr auto name = T::arg_name; -}; - -// Use return_name instead of name when available -template -struct as_return_type { - static constexpr auto name = T::name; -}; - -template -struct as_return_type::value>::type> { - static constexpr auto name = T::return_name; -}; - /// Helper class which loads arguments for C++ functions called from Python template class argument_loader { diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 84aaf9f702..e0f4cea2f9 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -155,6 +155,10 @@ struct handle_type_name> { template struct handle_type_name> { static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("list[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("list[") + as_return_type>::name + const_name("]"); }; template From b4008bbaa97e3cc4564f8d3b71d7c0127fc00641 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 17:25:28 +0100 Subject: [PATCH 03/32] Added support for arg/return type in stl/filesystem --- include/pybind11/stl/filesystem.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/pybind11/stl/filesystem.h b/include/pybind11/stl/filesystem.h index c16a9ae5c2..ecfb9cf0dc 100644 --- a/include/pybind11/stl/filesystem.h +++ b/include/pybind11/stl/filesystem.h @@ -107,6 +107,8 @@ struct path_caster { } PYBIND11_TYPE_CASTER(T, const_name("os.PathLike")); + static constexpr auto arg_name = const_name("Union[os.PathLike, str, bytes]"); + static constexpr auto return_name = const_name("Path"); }; #endif // PYBIND11_HAS_FILESYSTEM || defined(PYBIND11_HAS_EXPERIMENTAL_FILESYSTEM) From bc145b380bae8405eb4a3230649ed602cbfa0326 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 17:26:00 +0100 Subject: [PATCH 04/32] Added tests for arg/return type in stl/filesystem and py::typing::List --- tests/test_stl.cpp | 17 ++++++++++++++++- tests/test_stl.py | 15 ++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index dd93d51d0a..a3dc519cce 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -16,6 +16,7 @@ # define PYBIND11_HAS_FILESYSTEM_IS_OPTIONAL #endif #include +#include #include #include @@ -453,7 +454,21 @@ TEST_SUBMODULE(stl, m) { #ifdef PYBIND11_HAS_FILESYSTEM // test_fs_path m.attr("has_filesystem") = true; - m.def("parent_path", [](const std::filesystem::path &p) { return p.parent_path(); }); + m.def("parent_path", [](const std::filesystem::path &path) { return path.parent_path(); }); + m.def("parent_paths", [](const std::vector &paths) { + std::vector result; + for (const auto &path : paths) { + result.push_back(path.parent_path()); + } + return result; + }); + m.def("parent_paths_typing", [](py::typing::List paths) { + py::typing::List result; + for (auto path : paths) { + result.append(path.cast().parent_path()); + } + return result; + }); #endif #ifdef PYBIND11_TEST_VARIANT diff --git a/tests/test_stl.py b/tests/test_stl.py index d1a9ff08b0..9eaa862295 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -246,7 +246,7 @@ def test_reference_sensitive_optional(): @pytest.mark.skipif(not hasattr(m, "has_filesystem"), reason="no ") -def test_fs_path(): +def test_fs_path(doc): from pathlib import Path class PseudoStrPath: @@ -262,6 +262,19 @@ def __fspath__(self): assert m.parent_path(b"foo/bar") == Path("foo") assert m.parent_path(PseudoStrPath()) == Path("foo") assert m.parent_path(PseudoBytesPath()) == Path("foo") + assert ( + doc(m.parent_path) + == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> Path" + ) + assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] + assert ( + doc(m.parent_paths) + == "parent_paths(arg0: list[os.PathLike]) -> list[os.PathLike]" + ) + assert ( + doc(m.parent_paths_typing) + == "parent_paths_typing(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" + ) @pytest.mark.skipif(not hasattr(m, "load_variant"), reason="no ") From a468d379cff4ea902edf230dd4e5434d36c41229 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 18:17:18 +0100 Subject: [PATCH 05/32] Added arg/return name to more py::typing classes --- include/pybind11/typing.h | 100 ++++++++++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index e0f4cea2f9..26172f8589 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -128,9 +128,13 @@ PYBIND11_NAMESPACE_BEGIN(detail) template struct handle_type_name> { - static constexpr auto name = const_name("tuple[") - + ::pybind11::detail::concat(make_caster::name...) - + const_name("]"); + template + static constexpr auto wrap_name(const Ds &...names) { + return const_name("tuple[") + ::pybind11::detail::concat(names...) + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name...); + static constexpr auto arg_name = wrap_name(as_arg_type>::name...); + static constexpr auto return_name = wrap_name(as_return_type>::name...); }; template <> @@ -142,38 +146,70 @@ struct handle_type_name> { template struct handle_type_name> { // PEP 484 specifies this syntax for a variable-length tuple - static constexpr auto name - = const_name("tuple[") + make_caster::name + const_name(", ...]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("tuple[") + name + const_name(", ...]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") - + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const Kd &key_name, const Vd &value_name) { + return const_name("dict[") + key_name + const_name(", ") + value_name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name, make_caster::name); + static constexpr auto arg_name + = wrap_name(as_arg_type>::name, as_arg_type>::name); + static constexpr auto return_name + = wrap_name(as_return_type>::name, as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("list[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name - = const_name("list[") + as_return_type>::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("list[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("set[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("Iterable[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("Iterable[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("Iterator[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("Iterator[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template @@ -199,24 +235,46 @@ struct handle_type_name> { template struct handle_type_name> { - static constexpr auto name = const_name("Union[") - + ::pybind11::detail::concat(make_caster::name...) - + const_name("]"); + template + static constexpr auto wrap_name(const Ds &...names) { + return const_name("Union[") + ::pybind11::detail::concat(names...) + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name...); + static constexpr auto arg_name = wrap_name(as_arg_type>::name...); + static constexpr auto return_name = wrap_name(as_return_type>::name...); }; template struct handle_type_name> { - static constexpr auto name = const_name("Optional[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("Optional[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("TypeGuard[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("TypeGuard[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template struct handle_type_name> { - static constexpr auto name = const_name("TypeIs[") + make_caster::name + const_name("]"); + template + static constexpr auto wrap_name(const D &name) { + return const_name("TypeIs[") + name + const_name("]"); + } + static constexpr auto name = wrap_name(make_caster::name); + static constexpr auto arg_name = wrap_name(as_arg_type>::name); + static constexpr auto return_name = wrap_name(as_return_type>::name); }; template <> From f0b60cb795b61db8b1b005c6cc9d4932f13bd42e Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 18:23:22 +0100 Subject: [PATCH 06/32] Added arg/return type to Callable[...] --- include/pybind11/typing.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 26172f8589..8fa8d74aeb 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -216,16 +216,18 @@ template struct handle_type_name> { using retval_type = conditional_t::value, void_type, Return>; static constexpr auto name - = const_name("Callable[[") + ::pybind11::detail::concat(make_caster::name...) - + const_name("], ") + make_caster::name + const_name("]"); + = const_name("Callable[[") + + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("], ") + + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { // PEP 484 specifies this syntax for defining only return types of callables using retval_type = conditional_t::value, void_type, Return>; - static constexpr auto name - = const_name("Callable[..., ") + make_caster::name + const_name("]"); + static constexpr auto name = const_name("Callable[..., ") + + as_return_type>::name + + const_name("]"); }; template From 7f0f938a6acaa87b145a0fb2670e13835a155b13 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 18:56:06 +0100 Subject: [PATCH 07/32] Added tests for typing container classes (also nested) --- tests/test_stl.cpp | 28 +++++++++++++++++++++++++++- tests/test_stl.py | 39 +++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index a3dc519cce..255cb30017 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -462,13 +462,39 @@ TEST_SUBMODULE(stl, m) { } return result; }); - m.def("parent_paths_typing", [](py::typing::List paths) { + m.def("parent_paths_list", [](py::typing::List paths) { py::typing::List result; for (auto path : paths) { result.append(path.cast().parent_path()); } return result; }); + m.def("parent_paths_nested_list", + [](py::typing::List> paths_lists) { + py::typing::List> result_lists; + for (auto paths : paths_lists) { + py::typing::List result; + for (auto path : paths) { + result.append(path.cast().parent_path()); + } + result_lists.append(result); + } + return result_lists; + }); + m.def("parent_paths_tuple", + [](py::typing::Tuple paths) { + py::typing::Tuple result + = py::make_tuple(paths[0].cast().parent_path(), + paths[1].cast().parent_path()); + return result; + }); + m.def("parent_paths_dict", [](py::typing::Dict paths) { + py::typing::Dict result; + for (auto it : paths) { + result[it.first] = it.second.cast().parent_path(); + } + return result; + }); #endif #ifdef PYBIND11_TEST_VARIANT diff --git a/tests/test_stl.py b/tests/test_stl.py index 9eaa862295..14c7da312a 100644 --- a/tests/test_stl.py +++ b/tests/test_stl.py @@ -257,6 +257,7 @@ class PseudoBytesPath: def __fspath__(self): return b"foo/bar" + # Single argument assert m.parent_path(Path("foo/bar")) == Path("foo") assert m.parent_path("foo/bar") == Path("foo") assert m.parent_path(b"foo/bar") == Path("foo") @@ -266,14 +267,48 @@ def __fspath__(self): doc(m.parent_path) == "parent_path(arg0: Union[os.PathLike, str, bytes]) -> Path" ) + # std::vector should use name (for arg_name/return_name typing classes must be used) assert m.parent_paths(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] assert ( doc(m.parent_paths) == "parent_paths(arg0: list[os.PathLike]) -> list[os.PathLike]" ) + # py::typing::List + assert m.parent_paths_list(["foo/bar", "foo/baz"]) == [Path("foo"), Path("foo")] assert ( - doc(m.parent_paths_typing) - == "parent_paths_typing(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" + doc(m.parent_paths_list) + == "parent_paths_list(arg0: list[Union[os.PathLike, str, bytes]]) -> list[Path]" + ) + # Nested py::typing::List + assert m.parent_paths_nested_list([["foo/bar"], ["foo/baz", "foo/buzz"]]) == [ + [Path("foo")], + [Path("foo"), Path("foo")], + ] + assert ( + doc(m.parent_paths_nested_list) + == "parent_paths_nested_list(arg0: list[list[Union[os.PathLike, str, bytes]]]) -> list[list[Path]]" + ) + # py::typing::Tuple + assert m.parent_paths_tuple(("foo/bar", "foo/baz")) == (Path("foo"), Path("foo")) + assert ( + doc(m.parent_paths_tuple) + == "parent_paths_tuple(arg0: tuple[Union[os.PathLike, str, bytes], Union[os.PathLike, str, bytes]]) -> tuple[Path, Path]" + ) + # py::typing::Dict + assert m.parent_paths_dict( + { + "key1": Path("foo/bar"), + "key2": "foo/baz", + "key3": b"foo/buzz", + } + ) == { + "key1": Path("foo"), + "key2": Path("foo"), + "key3": Path("foo"), + } + assert ( + doc(m.parent_paths_dict) + == "parent_paths_dict(arg0: dict[str, Union[os.PathLike, str, bytes]]) -> dict[str, Path]" ) From e8d94ea2a8bd11478f3941b934aa8106c56ee12a Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 19:45:06 +0100 Subject: [PATCH 08/32] Changed typing classes to avoid using C++14 auto return type deduction. --- include/pybind11/typing.h | 148 ++++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 79 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 8fa8d74aeb..17a65927fa 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -128,13 +128,16 @@ PYBIND11_NAMESPACE_BEGIN(detail) template struct handle_type_name> { - template - static constexpr auto wrap_name(const Ds &...names) { - return const_name("tuple[") + ::pybind11::detail::concat(names...) + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name...); - static constexpr auto arg_name = wrap_name(as_arg_type>::name...); - static constexpr auto return_name = wrap_name(as_return_type>::name...); + static constexpr auto name = const_name("tuple[") + + ::pybind11::detail::concat(make_caster::name...) + + const_name("]"); + static constexpr auto arg_name + = const_name("tuple[") + + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("]"); + static constexpr auto return_name + = const_name("tuple[") + + ::pybind11::detail::concat(as_return_type>::name...) + + const_name("]"); }; template <> @@ -146,70 +149,60 @@ struct handle_type_name> { template struct handle_type_name> { // PEP 484 specifies this syntax for a variable-length tuple - template - static constexpr auto wrap_name(const D &name) { - return const_name("tuple[") + name + const_name(", ...]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name + = const_name("tuple[") + make_caster::name + const_name(", ...]"); + static constexpr auto arg_name + = const_name("tuple[") + as_arg_type>::name + const_name(", ...]"); + static constexpr auto return_name + = const_name("tuple[") + as_return_type>::name + const_name(", ...]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const Kd &key_name, const Vd &value_name) { - return const_name("dict[") + key_name + const_name(", ") + value_name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name, make_caster::name); - static constexpr auto arg_name - = wrap_name(as_arg_type>::name, as_arg_type>::name); - static constexpr auto return_name - = wrap_name(as_return_type>::name, as_return_type>::name); + static constexpr auto name = const_name("dict[") + make_caster::name + const_name(", ") + + make_caster::name + const_name("]"); + static constexpr auto arg_name = const_name("dict[") + as_arg_type>::name + + const_name(", ") + as_arg_type>::name + + const_name("]"); + static constexpr auto return_name = const_name("dict[") + as_return_type>::name + + const_name(", ") + as_return_type>::name + + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("list[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("list[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("list[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("list[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("set[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("set[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("set[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("set[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("Iterable[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("Iterable[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("Iterable[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("Iterable[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("Iterator[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("Iterator[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("Iterator[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("Iterator[") + as_return_type>::name + const_name("]"); }; template @@ -237,46 +230,43 @@ struct handle_type_name> { template struct handle_type_name> { - template - static constexpr auto wrap_name(const Ds &...names) { - return const_name("Union[") + ::pybind11::detail::concat(names...) + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name...); - static constexpr auto arg_name = wrap_name(as_arg_type>::name...); - static constexpr auto return_name = wrap_name(as_return_type>::name...); + static constexpr auto name = const_name("Union[") + + ::pybind11::detail::concat(make_caster::name...) + + const_name("]"); + static constexpr auto arg_name + = const_name("Union[") + + ::pybind11::detail::concat(as_arg_type>::name...) + const_name("]"); + static constexpr auto return_name + = const_name("Union[") + + ::pybind11::detail::concat(as_return_type>::name...) + + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("Optional[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("Optional[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("Optional[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("Optional[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("TypeGuard[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("TypeGuard[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("TypeGuard[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("TypeGuard[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - template - static constexpr auto wrap_name(const D &name) { - return const_name("TypeIs[") + name + const_name("]"); - } - static constexpr auto name = wrap_name(make_caster::name); - static constexpr auto arg_name = wrap_name(as_arg_type>::name); - static constexpr auto return_name = wrap_name(as_return_type>::name); + static constexpr auto name = const_name("TypeIs[") + make_caster::name + const_name("]"); + static constexpr auto arg_name + = const_name("TypeIs[") + as_arg_type>::name + const_name("]"); + static constexpr auto return_name + = const_name("TypeIs[") + as_return_type>::name + const_name("]"); }; template <> From 90428443bde7a50b01d9c3f8ed726b932e98cd90 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 20:21:14 +0100 Subject: [PATCH 09/32] Fixed clang-tidy errors. --- tests/test_stl.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 255cb30017..3a378e085c 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -457,12 +457,13 @@ TEST_SUBMODULE(stl, m) { m.def("parent_path", [](const std::filesystem::path &path) { return path.parent_path(); }); m.def("parent_paths", [](const std::vector &paths) { std::vector result; + result.reserve(paths.size()); for (const auto &path : paths) { result.push_back(path.parent_path()); } return result; }); - m.def("parent_paths_list", [](py::typing::List paths) { + m.def("parent_paths_list", [](const py::typing::List &paths) { py::typing::List result; for (auto path : paths) { result.append(path.cast().parent_path()); @@ -470,7 +471,7 @@ TEST_SUBMODULE(stl, m) { return result; }); m.def("parent_paths_nested_list", - [](py::typing::List> paths_lists) { + [](const py::typing::List> &paths_lists) { py::typing::List> result_lists; for (auto paths : paths_lists) { py::typing::List result; @@ -482,19 +483,20 @@ TEST_SUBMODULE(stl, m) { return result_lists; }); m.def("parent_paths_tuple", - [](py::typing::Tuple paths) { + [](const py::typing::Tuple &paths) { py::typing::Tuple result = py::make_tuple(paths[0].cast().parent_path(), paths[1].cast().parent_path()); return result; }); - m.def("parent_paths_dict", [](py::typing::Dict paths) { - py::typing::Dict result; - for (auto it : paths) { - result[it.first] = it.second.cast().parent_path(); - } - return result; - }); + m.def("parent_paths_dict", + [](const py::typing::Dict &paths) { + py::typing::Dict result; + for (auto it : paths) { + result[it.first] = it.second.cast().parent_path(); + } + return result; + }); #endif #ifdef PYBIND11_TEST_VARIANT From c7bcb57410acb0ddfbc05d184f912778201e45d1 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 21:50:58 +0100 Subject: [PATCH 10/32] Changed Enable to SFINAE --- include/pybind11/cast.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/pybind11/cast.h b/include/pybind11/cast.h index 3088a518a8..8aa238db77 100644 --- a/include/pybind11/cast.h +++ b/include/pybind11/cast.h @@ -45,7 +45,7 @@ template struct is_descr> : std::true_type {}; // Use arg_name instead of name when available -template +template struct as_arg_type { static constexpr auto name = T::name; }; @@ -56,7 +56,7 @@ struct as_arg_type::v }; // Use return_name instead of name when available -template +template struct as_return_type { static constexpr auto name = T::name; }; From 251aeb7e1a3a7549afd148ce39b2690d2184df3f Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 24 Nov 2024 21:51:28 +0100 Subject: [PATCH 11/32] Added test for Tuple[T, ...] --- tests/test_stl.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_stl.cpp b/tests/test_stl.cpp index 3a378e085c..9ddd951e0c 100644 --- a/tests/test_stl.cpp +++ b/tests/test_stl.cpp @@ -489,6 +489,14 @@ TEST_SUBMODULE(stl, m) { paths[1].cast().parent_path()); return result; }); + m.def("parent_paths_tuple_ellipsis", + [](const py::typing::Tuple &paths) { + py::typing::Tuple result(paths.size()); + for (size_t i = 0; i < paths.size(); ++i) { + result[i] = paths[i].cast().parent_path(); + } + return result; + }); m.def("parent_paths_dict", [](const py::typing::Dict &paths) { py::typing::Dict result; From 413d685aae32c7fe8a8967a8d2fc6aff9daf0043 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 01:22:44 +0100 Subject: [PATCH 12/32] Added RealNumber with custom caster for testing typing classes. --- tests/test_pytypes.cpp | 79 ++++++++++++++++++++++++++++++++++++++++++ tests/test_pytypes.py | 29 ++++++++++++++++ 2 files changed, 108 insertions(+) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 8df4cdd3f6..abd49c29e3 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -137,6 +137,44 @@ typedef py::typing::TypeVar<"V"> TypeVarV; } // namespace typevar #endif +// Custom type for testing arg_name/return_name type hints +// RealNumber: +// in arguments -> float | int, +// in return -> float +// fallback -> complex (just for testing, not really useful here) + +struct RealNumber { + double value; +}; + +namespace pybind11 { +namespace detail { + +template <> +struct type_caster { + PYBIND11_TYPE_CASTER(RealNumber, const_name("complex")); + static constexpr auto arg_name = const_name("Union[float, int]"); + static constexpr auto return_name = const_name("float"); + + static handle cast(const RealNumber &number, return_value_policy, handle) { + return PyFloat_FromDouble(number.value); + } + + bool load(handle src, bool) { + if (!src) { + return false; + } + if (!PyFloat_Check(src.ptr()) && !PyLong_Check(src.ptr())) { + return false; + } + value = RealNumber{PyFloat_AsDouble(src.ptr())}; + return true; + } +}; + +} // namespace detail +} // namespace pybind11 + TEST_SUBMODULE(pytypes, m) { m.def("obj_class_name", [](py::handle obj) { return py::detail::obj_class_name(obj.ptr()); }); @@ -998,4 +1036,45 @@ TEST_SUBMODULE(pytypes, m) { #else m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = false; #endif + m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; }); + m.def("half_of_number_tuple", [](const py::typing::Tuple &x) { + py::typing::Tuple result + = py::make_tuple(RealNumber{x[0].cast().value / 2}, + RealNumber{x[1].cast().value / 2}); + return result; + }); + m.def("half_of_number_tuple_ellipsis", + [](const py::typing::Tuple &x) { + py::typing::Tuple result(x.size()); + for (size_t i = 0; i < x.size(); ++i) { + result[i] = x[i].cast().value / 2; + } + return result; + }); + m.def("half_of_number_list", [](const py::typing::List &x) { + py::typing::List result; + for (auto num : x) { + result.append(RealNumber{num.cast().value / 2}); + } + return result; + }); + m.def("half_of_number_nested_list", + [](const py::typing::List> &x) { + py::typing::List> result_lists; + for (auto nums : x) { + py::typing::List result; + for (auto num : nums) { + result.append(RealNumber{num.cast().value / 2}); + } + result_lists.append(result); + } + return result_lists; + }); + m.def("half_of_number_dict", [](const py::typing::Dict &x) { + py::typing::Dict result; + for (auto it : x) { + result[it.first] = RealNumber{it.second.cast().value / 2}; + } + return result; + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 9fd24b34f1..6dd01ee31e 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1101,3 +1101,32 @@ def test_list_ranges(tested_list, expected): def test_dict_ranges(tested_dict, expected): assert m.dict_iterator_default_initialization() assert m.transform_dict_plus_one(tested_dict) == expected + + +def test_arg_return_type_hints(doc): + assert doc(m.half_of_number) == "half_of_number(arg0: Union[float, int]) -> float" + assert m.half_of_number(2.0) == 1.0 + assert m.half_of_number(2) == 1.0 + assert m.half_of_number(0) == 0 + assert isinstance(m.half_of_number(0), float) + assert not isinstance(m.half_of_number(0), int) + assert ( + doc(m.half_of_number_tuple) + == "half_of_number_tuple(arg0: tuple[Union[float, int], Union[float, int]]) -> tuple[float, float]" + ) + assert ( + doc(m.half_of_number_tuple_ellipsis) + == "half_of_number_tuple_ellipsis(arg0: tuple[Union[float, int], ...]) -> tuple[float, ...]" + ) + assert ( + doc(m.half_of_number_list) + == "half_of_number_list(arg0: list[Union[float, int]]) -> list[float]" + ) + assert ( + doc(m.half_of_number_nested_list) + == "half_of_number_nested_list(arg0: list[list[Union[float, int]]]) -> list[list[float]]" + ) + assert ( + doc(m.half_of_number_dict) + == "half_of_number_dict(arg0: dict[str, Union[float, int]]) -> dict[str, float]" + ) From 66e6644dd422d9f6d827284a980963069f68a758 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 11:14:59 +0100 Subject: [PATCH 13/32] Added tests for Set, Iterable, Iterator, Union, and Optional --- tests/test_pytypes.cpp | 40 +++++++++++++++++++++++++++++++++------- tests/test_pytypes.py | 38 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 69 insertions(+), 9 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index abd49c29e3..bc7e3a37e7 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1037,12 +1037,14 @@ TEST_SUBMODULE(pytypes, m) { m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = false; #endif m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; }); + // Tuple m.def("half_of_number_tuple", [](const py::typing::Tuple &x) { py::typing::Tuple result = py::make_tuple(RealNumber{x[0].cast().value / 2}, RealNumber{x[1].cast().value / 2}); return result; }); + // Tuple m.def("half_of_number_tuple_ellipsis", [](const py::typing::Tuple &x) { py::typing::Tuple result(x.size()); @@ -1051,6 +1053,15 @@ TEST_SUBMODULE(pytypes, m) { } return result; }); + // Dict + m.def("half_of_number_dict", [](const py::typing::Dict &x) { + py::typing::Dict result; + for (auto it : x) { + result[it.first] = RealNumber{it.second.cast().value / 2}; + } + return result; + }); + // List m.def("half_of_number_list", [](const py::typing::List &x) { py::typing::List result; for (auto num : x) { @@ -1058,6 +1069,7 @@ TEST_SUBMODULE(pytypes, m) { } return result; }); + // List> m.def("half_of_number_nested_list", [](const py::typing::List> &x) { py::typing::List> result_lists; @@ -1070,11 +1082,25 @@ TEST_SUBMODULE(pytypes, m) { } return result_lists; }); - m.def("half_of_number_dict", [](const py::typing::Dict &x) { - py::typing::Dict result; - for (auto it : x) { - result[it.first] = RealNumber{it.second.cast().value / 2}; - } - return result; - }); + // Set + m.def("identity_set", [](const py::typing::Set &x) { return x; }); + // Iterable + m.def("identity_iterable", [](const py::typing::Iterable &x) { return x; }); + // Iterator + m.def("identity_iterator", [](const py::typing::Iterator &x) { return x; }); + // Callable + // m.def("get_identity_callable", []() -> py::typing::Callable + // { return [](const RealNumber &x) { return x; }; + // }); + // Callable + // m.def("get_identity_callable_only_return", + // []() -> py::typing::Callable { + // return [](const RealNumber &x) { return x; }; + // }); + // Union + m.def("identity_union", [](const py::typing::Union &x) { return x; }); + // Optional + m.def("identity_optional", [](const py::typing::Optional &x) { return x; }); + // TypeGuard + // TypeIs } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 6dd01ee31e..35b080c922 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1110,23 +1110,57 @@ def test_arg_return_type_hints(doc): assert m.half_of_number(0) == 0 assert isinstance(m.half_of_number(0), float) assert not isinstance(m.half_of_number(0), int) + # Tuple assert ( doc(m.half_of_number_tuple) == "half_of_number_tuple(arg0: tuple[Union[float, int], Union[float, int]]) -> tuple[float, float]" ) + # Tuple assert ( doc(m.half_of_number_tuple_ellipsis) == "half_of_number_tuple_ellipsis(arg0: tuple[Union[float, int], ...]) -> tuple[float, ...]" ) + # Dict + assert ( + doc(m.half_of_number_dict) + == "half_of_number_dict(arg0: dict[str, Union[float, int]]) -> dict[str, float]" + ) + # List assert ( doc(m.half_of_number_list) == "half_of_number_list(arg0: list[Union[float, int]]) -> list[float]" ) + # List> assert ( doc(m.half_of_number_nested_list) == "half_of_number_nested_list(arg0: list[list[Union[float, int]]]) -> list[list[float]]" ) + # Set assert ( - doc(m.half_of_number_dict) - == "half_of_number_dict(arg0: dict[str, Union[float, int]]) -> dict[str, float]" + doc(m.identity_set) + == "identity_set(arg0: set[Union[float, int]]) -> set[float]" + ) + # Iterable + assert ( + doc(m.identity_iterable) + == "identity_iterable(arg0: Iterable[Union[float, int]]) -> Iterable[float]" + ) + # Iterator + assert ( + doc(m.identity_iterator) + == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]" + ) + # Callable + # Callable + # Union + assert ( + doc(m.identity_union) + == "identity_union(arg0: Union[Union[float, int], str]) -> Union[float, str]" + ) + # Optional + assert ( + doc(m.identity_optional) + == "identity_optional(arg0: Optional[Union[float, int]]) -> Optional[float]" ) + # TypeGuard + # TypeIs From 2c17048b74781f70f76e8e8dd6c46e22d29ec740 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 12:09:34 +0100 Subject: [PATCH 14/32] Added tests for Callable --- tests/test_pytypes.cpp | 15 ++++++++------- tests/test_pytypes.py | 8 ++++++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index bc7e3a37e7..ab1cd01a95 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1089,14 +1089,15 @@ TEST_SUBMODULE(pytypes, m) { // Iterator m.def("identity_iterator", [](const py::typing::Iterator &x) { return x; }); // Callable - // m.def("get_identity_callable", []() -> py::typing::Callable - // { return [](const RealNumber &x) { return x; }; - // }); + m.def("apply_callable", + [](const RealNumber &x, const py::typing::Callable &f) { + return f(x).cast(); + }); // Callable - // m.def("get_identity_callable_only_return", - // []() -> py::typing::Callable { - // return [](const RealNumber &x) { return x; }; - // }); + m.def("apply_callable", + [](const RealNumber &x, const py::typing::Callable &f) { + return f(x).cast(); + }); // Union m.def("identity_union", [](const py::typing::Union &x) { return x; }); // Optional diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 35b080c922..f5ae0be5f7 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1151,7 +1151,15 @@ def test_arg_return_type_hints(doc): == "identity_iterator(arg0: Iterator[Union[float, int]]) -> Iterator[float]" ) # Callable + assert ( + doc(m.apply_callable) + == "apply_callable(arg0: Union[float, int], arg1: Callable[[Union[float, int]], float]) -> float" + ) # Callable + assert ( + doc(m.apply_callable) + == "apply_callable(arg0: Union[float, int], arg1: Callable[[...], float]) -> float" + ) # Union assert ( doc(m.identity_union) From 9eb7af9eb1ddbcbc5dca018332ec830d41da3c09 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 13:25:28 +0100 Subject: [PATCH 15/32] Fixed Callable with ellipsis test --- tests/test_pytypes.cpp | 2 +- tests/test_pytypes.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index ab1cd01a95..c964cc08d4 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1094,7 +1094,7 @@ TEST_SUBMODULE(pytypes, m) { return f(x).cast(); }); // Callable - m.def("apply_callable", + m.def("apply_callable_ellipsis", [](const RealNumber &x, const py::typing::Callable &f) { return f(x).cast(); }); diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index f5ae0be5f7..e10f325fb4 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1157,8 +1157,8 @@ def test_arg_return_type_hints(doc): ) # Callable assert ( - doc(m.apply_callable) - == "apply_callable(arg0: Union[float, int], arg1: Callable[[...], float]) -> float" + doc(m.apply_callable_ellipsis) + == "apply_callable_ellipsis(arg0: Union[float, int], arg1: Callable[..., float]) -> float" ) # Union assert ( From 9ad445d93b55d1d3863fa10132a4f7e617811ef8 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 13:26:42 +0100 Subject: [PATCH 16/32] Changed TypeGuard/TypeIs to use return type (being the narrower type) + Tests --- include/pybind11/typing.h | 13 +++++-------- tests/test_pytypes.cpp | 13 +++++++++++++ tests/test_pytypes.py | 8 ++++++++ 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/include/pybind11/typing.h b/include/pybind11/typing.h index 17a65927fa..405ff8714a 100644 --- a/include/pybind11/typing.h +++ b/include/pybind11/typing.h @@ -251,21 +251,18 @@ struct handle_type_name> { = const_name("Optional[") + as_return_type>::name + const_name("]"); }; +// TypeGuard and TypeIs use as_return_type to use the return type if available, which is usually +// the narrower type. + template struct handle_type_name> { - static constexpr auto name = const_name("TypeGuard[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("TypeGuard[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name + static constexpr auto name = const_name("TypeGuard[") + as_return_type>::name + const_name("]"); }; template struct handle_type_name> { - static constexpr auto name = const_name("TypeIs[") + make_caster::name + const_name("]"); - static constexpr auto arg_name - = const_name("TypeIs[") + as_arg_type>::name + const_name("]"); - static constexpr auto return_name + static constexpr auto name = const_name("TypeIs[") + as_return_type>::name + const_name("]"); }; diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index c964cc08d4..8e39735262 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -1103,5 +1103,18 @@ TEST_SUBMODULE(pytypes, m) { // Optional m.def("identity_optional", [](const py::typing::Optional &x) { return x; }); // TypeGuard + m.def("check_type_guard", + [](const py::typing::List &x) + -> py::typing::TypeGuard> { + for (const auto &item : x) { + if (!py::isinstance(item)) { + return false; + } + } + return true; + }); // TypeIs + m.def("check_type_is", [](const py::object &x) -> py::typing::TypeIs { + return py::isinstance(x); + }); } diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index e10f325fb4..429ce4a8e6 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1171,4 +1171,12 @@ def test_arg_return_type_hints(doc): == "identity_optional(arg0: Optional[Union[float, int]]) -> Optional[float]" ) # TypeGuard + assert ( + doc(m.check_type_guard) + == "check_type_guard(arg0: list[object]) -> TypeGuard[list[float]]" + ) # TypeIs + assert ( + doc(m.check_type_is) + == "check_type_is(arg0: object) -> TypeIs[float]" + ) From c9dab34a2868e8d1b128e84f280d28453e12ef1d Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 25 Nov 2024 13:41:08 +0100 Subject: [PATCH 17/32] Added test for use of fallback type name with stl vector --- tests/test_pytypes.cpp | 10 ++++++++++ tests/test_pytypes.py | 10 ++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 8e39735262..7dab0de475 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -7,6 +7,7 @@ BSD-style license that can be found in the LICENSE file. */ +#include #include #include "pybind11_tests.h" @@ -1037,6 +1038,15 @@ TEST_SUBMODULE(pytypes, m) { m.attr("defined_PYBIND11_TEST_PYTYPES_HAS_RANGES") = false; #endif m.def("half_of_number", [](const RealNumber &x) { return RealNumber{x.value / 2}; }); + // std::vector + m.def("half_of_number_vector", [](const std::vector &x) { + std::vector result; + result.reserve(x.size()); + for (auto num : x) { + result.push_back(RealNumber{num.value / 2}); + } + return result; + }); // Tuple m.def("half_of_number_tuple", [](const py::typing::Tuple &x) { py::typing::Tuple result diff --git a/tests/test_pytypes.py b/tests/test_pytypes.py index 429ce4a8e6..b6e64b9bf6 100644 --- a/tests/test_pytypes.py +++ b/tests/test_pytypes.py @@ -1110,6 +1110,11 @@ def test_arg_return_type_hints(doc): assert m.half_of_number(0) == 0 assert isinstance(m.half_of_number(0), float) assert not isinstance(m.half_of_number(0), int) + # std::vector should use fallback type (complex is not really useful but just used for testing) + assert ( + doc(m.half_of_number_vector) + == "half_of_number_vector(arg0: list[complex]) -> list[complex]" + ) # Tuple assert ( doc(m.half_of_number_tuple) @@ -1176,7 +1181,4 @@ def test_arg_return_type_hints(doc): == "check_type_guard(arg0: list[object]) -> TypeGuard[list[float]]" ) # TypeIs - assert ( - doc(m.check_type_is) - == "check_type_is(arg0: object) -> TypeIs[float]" - ) + assert doc(m.check_type_is) == "check_type_is(arg0: object) -> TypeIs[float]" From eb8e5d1c9a1e59c325f53f021e385412a2afeb97 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 26 Nov 2024 16:19:59 +0100 Subject: [PATCH 18/32] Updated documentation. --- docs/advanced/cast/custom.rst | 166 +++++++++++++++-------- tests/CMakeLists.txt | 1 + tests/test_docs_advanced_cast_custom.cpp | 79 +++++++++++ tests/test_docs_advanced_cast_custom.py | 37 +++++ 4 files changed, 224 insertions(+), 59 deletions(-) create mode 100644 tests/test_docs_advanced_cast_custom.cpp create mode 100644 tests/test_docs_advanced_cast_custom.py diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 8138cac619..6a4f6e2a23 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -1,35 +1,52 @@ Custom type casters =================== -In very rare cases, applications may require custom type casters that cannot be -expressed using the abstractions provided by pybind11, thus requiring raw -Python C API calls. This is fairly advanced usage and should only be pursued by -experts who are familiar with the intricacies of Python reference counting. - -The following snippets demonstrate how this works for a very simple ``inty`` -type that that should be convertible from Python types that provide a -``__int__(self)`` method. +Some applications may prefer custom type casters that convert between existing +Python types and C++ types, similar to the ``list`` ↔ ``std::vector`` +and ``dict`` ↔ ``std::map`` conversions which are built into pybind11. +Implementing custom type casters is fairly advanced usage and requires +familiarity with the intricacies of the Python C API. +You can refer to the `Python/C API Reference Manual `_ +for more information. + +The following snippets demonstrate how this works for a very simple ``Point2D`` type. +We want this type to be convertible to C++ from Python types implementing the +``Sequence`` protocol and having two elements of type ``float``. +When returned from C++ to Python, it should be converted to a Python ``tuple[float, float]``. +For this type we could provide Python bindings for different arithmetic functions implemented +in C++ (here demonstrated by a simple ``negate`` function). + +.. + PLEASE KEEP THE CODE BLOCKS IN SYNC WITH + tests/test_docs_advanced_cast_custom.cpp + tests/test_docs_advanced_cast_custom.py + Ideally, change the test, run pre-commit (incl. clang-format), + then copy the changed code back here. + Also use TEST_SUBMODULE in tests, but PYBIND11_MODULE in docs. .. code-block:: cpp - struct inty { long long_value; }; + namespace user_space { - void print(inty s) { - std::cout << s.long_value << std::endl; - } + struct Point2D { + double x; + double y; + }; -The following Python snippet demonstrates the intended usage from the Python side: + Point2D negate(const Point2D &point) { return Point2D{-point.x, -point.y}; } -.. code-block:: python + } // namespace user_space - class A: - def __int__(self): - return 123 +The following Python snippet demonstrates the intended usage of ``negate`` from the Python side: + +.. code-block:: python - from example import print + from my_math_module import docs_advanced_cast_custom as m - print(A()) + point1 = [1.0, -1.0] + point2 = m.negate(point1) + assert point2 == (-1.0, 1.0) To register the necessary conversion routines, it is necessary to add an instantiation of the ``pybind11::detail::type_caster`` template. @@ -38,47 +55,68 @@ type is explicitly allowed. .. code-block:: cpp - namespace PYBIND11_NAMESPACE { namespace detail { - template <> struct type_caster { - public: - /** - * This macro establishes the name 'inty' in - * function signatures and declares a local variable - * 'value' of type inty - */ - PYBIND11_TYPE_CASTER(inty, const_name("inty")); - - /** - * Conversion part 1 (Python->C++): convert a PyObject into a inty - * instance or return false upon failure. The second argument - * indicates whether implicit conversions should be applied. - */ - bool load(handle src, bool) { - /* Extract PyObject from handle */ - PyObject *source = src.ptr(); - /* Try converting into a Python integer value */ - PyObject *tmp = PyNumber_Long(source); - if (!tmp) - return false; - /* Now try to convert into a C++ int */ - value.long_value = PyLong_AsLong(tmp); - Py_DECREF(tmp); - /* Ensure return code was OK (to avoid out-of-range errors etc) */ - return !(value.long_value == -1 && !PyErr_Occurred()); + namespace pybind11 { + namespace detail { + + template <> + struct type_caster { + // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple` + PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); + // `arg_name` and `return_name` may optionally be used to specify type hints separately for + // arguments and return values. + // The signature of our identity function would then look like: + // `identity(Sequence[float]) -> tuple[float, float]` + static constexpr auto arg_name = const_name("Sequence[float]"); + static constexpr auto return_name = const_name("tuple[float, float]"); + + // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + static handle cast(const user_space::Point2D &number, return_value_policy, handle) { + auto x = PyFloat_FromDouble(number.x); + if (!x) { + return nullptr; } - - /** - * Conversion part 2 (C++ -> Python): convert an inty instance into - * a Python object. The second and third arguments are used to - * indicate the return value policy and parent object (for - * ``return_value_policy::reference_internal``) and are generally - * ignored by implicit casters. - */ - static handle cast(inty src, return_value_policy /* policy */, handle /* parent */) { - return PyLong_FromLong(src.long_value); + auto y = PyFloat_FromDouble(number.y); + if (!y) { + Py_DECREF(x); + return nullptr; + } + return PyTuple_Pack(2, x, y); + } + + // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + bool load(handle src, bool) { + // Check if handle is valid Sequence of length 2 + if (!src || !PySequence_Check(src.ptr()) || PySequence_Length(src.ptr()) != 2) { + return false; + } + auto x = PySequence_GetItem(src.ptr(), 0); + auto y = PySequence_GetItem(src.ptr(), 1); + // Check if values are float or int (both are allowed with float as type hint) + if (!x || !(PyFloat_Check(x) || PyLong_Check(x))) { + return false; } - }; - }} // namespace PYBIND11_NAMESPACE::detail + if (!y || !(PyFloat_Check(y) || PyLong_Check(y))) { + return false; + } + // value is a default constructed Point2D + value.x = PyFloat_AsDouble(x); + value.y = PyFloat_AsDouble(y); + if (PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + return true; + } + }; + + } // namespace detail + } // namespace pybind11 + + // Bind the negate function + PYBIND11_MODULE(docs_advanced_cast_custom, m) { m.def("negate", user_space::negate); } .. note:: @@ -89,5 +127,15 @@ type is explicitly allowed. .. warning:: When using custom type casters, it's important to declare them consistently - in every compilation unit of the Python extension module. Otherwise, + in every compilation unit of the Python extension module to satisfy the C++ One Definition Rule + (`ODR `_).. Otherwise, undefined behavior can ensue. + +.. note:: + + Using the type hint ``Sequence[float]`` signals to static type checkers, that not only tuples may be + passed, but any type implementing the Sequence protocol, e.g., ``list[float]``. + Unfortunately, that loses the length information ``tuple[float, float]`` provides. + One way of still providing some length information in type hints is using ``typing.Annotated``, e.g., + ``Annotated[Sequence[float], 2]``, or further add libraries like + `annotated-types `. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 01b6c0a3e6..315a5bad04 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -122,6 +122,7 @@ set(PYBIND11_TEST_FILES test_custom_type_casters test_custom_type_setup test_docstring_options + test_docs_advanced_cast_custom test_eigen_matrix test_eigen_tensor test_enum diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp new file mode 100644 index 0000000000..5b2459b2f4 --- /dev/null +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -0,0 +1,79 @@ +// ######################################################################### +// PLEASE UPDATE docs/advanced/cast/custom.rst IF ANY CHANGES ARE MADE HERE. +// ######################################################################### + +#include "pybind11_tests.h" + +namespace user_space { + +struct Point2D { + double x; + double y; +}; + +Point2D negate(const Point2D &point) { return Point2D{-point.x, -point.y}; } + +} // namespace user_space + +namespace pybind11 { +namespace detail { + +template <> +struct type_caster { + // This macro inserts a lot of boilerplate code and sets the default type hint to `tuple` + PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); + // `arg_name` and `return_name` may optionally be used to specify type hints separately for + // arguments and return values. + // The signature of our identity function would then look like: + // `identity(Sequence[float]) -> tuple[float, float]` + static constexpr auto arg_name = const_name("Sequence[float]"); + static constexpr auto return_name = const_name("tuple[float, float]"); + + // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments + // are used to indicate the return value policy and parent object (for + // return_value_policy::reference_internal) and are often ignored by custom casters. + static handle cast(const user_space::Point2D &number, return_value_policy, handle) { + auto x = PyFloat_FromDouble(number.x); + if (!x) { + return nullptr; + } + auto y = PyFloat_FromDouble(number.y); + if (!y) { + Py_DECREF(x); + return nullptr; + } + return PyTuple_Pack(2, x, y); + } + + // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The + // second argument indicates whether implicit conversions should be allowed. + bool load(handle src, bool) { + // Check if handle is valid Sequence of length 2 + if (!src || !PySequence_Check(src.ptr()) || PySequence_Length(src.ptr()) != 2) { + return false; + } + auto x = PySequence_GetItem(src.ptr(), 0); + auto y = PySequence_GetItem(src.ptr(), 1); + // Check if values are float or int (both are allowed with float as type hint) + if (!x || !(PyFloat_Check(x) || PyLong_Check(x))) { + return false; + } + if (!y || !(PyFloat_Check(y) || PyLong_Check(y))) { + return false; + } + // value is a default constructed Point2D + value.x = PyFloat_AsDouble(x); + value.y = PyFloat_AsDouble(y); + if (PyErr_Occurred()) { + PyErr_Clear(); + return false; + } + return true; + } +}; + +} // namespace detail +} // namespace pybind11 + +// Bind the negate function +TEST_SUBMODULE(docs_advanced_cast_custom, m) { m.def("negate", user_space::negate); } diff --git a/tests/test_docs_advanced_cast_custom.py b/tests/test_docs_advanced_cast_custom.py new file mode 100644 index 0000000000..8018b8f576 --- /dev/null +++ b/tests/test_docs_advanced_cast_custom.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Sequence + +if TYPE_CHECKING: + from conftest import SanitizedString + +from pybind11_tests import docs_advanced_cast_custom as m + + +def assert_negate_function( + input_sequence: Sequence[float], + target: tuple[float, float], +) -> None: + output = m.negate(input_sequence) + assert isinstance(output, tuple) + assert len(output) == 2 + assert isinstance(output[0], float) + assert isinstance(output[1], float) + assert output == target + + +def test_negate(doc: SanitizedString) -> None: + assert doc(m.negate) == "negate(arg0: Sequence[float]) -> tuple[float, float]" + assert_negate_function([1.0, -1.0], (-1.0, 1.0)) + assert_negate_function((1.0, -1.0), (-1.0, 1.0)) + assert_negate_function([1, -1], (-1.0, 1.0)) + assert_negate_function((1, -1), (-1.0, 1.0)) + + +def test_docs() -> None: + ########################################################################### + # PLEASE UPDATE docs/advanced/cast/custom.rst IF ANY CHANGES ARE MADE HERE. + ########################################################################### + point1 = [1.0, -1.0] + point2 = m.negate(point1) + assert point2 == (-1.0, 1.0) From 3c53525c66a39867cf5acaf88ba3d4fc811570e3 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 26 Nov 2024 16:21:02 +0100 Subject: [PATCH 19/32] Fixed unnecessary constructor call in test. --- tests/test_pytypes.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index 7dab0de475..f2c39160f6 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -168,7 +168,7 @@ struct type_caster { if (!PyFloat_Check(src.ptr()) && !PyLong_Check(src.ptr())) { return false; } - value = RealNumber{PyFloat_AsDouble(src.ptr())}; + value.value = PyFloat_AsDouble(src.ptr()); return true; } }; From a45f1bd25e6379428b2db2f3f85ba2f4d58004ac Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 26 Nov 2024 17:01:53 +0100 Subject: [PATCH 20/32] Fixed reference counting in example type caster. --- docs/advanced/cast/custom.rst | 31 +++++++++++++++--------- tests/test_docs_advanced_cast_custom.cpp | 29 +++++++++++++--------- 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 6a4f6e2a23..83aafbc6ec 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -73,16 +73,21 @@ type is explicitly allowed. // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. static handle cast(const user_space::Point2D &number, return_value_policy, handle) { + // Convert x and y components to python float auto x = PyFloat_FromDouble(number.x); - if (!x) { - return nullptr; - } auto y = PyFloat_FromDouble(number.y); - if (!y) { - Py_DECREF(x); + // Check if conversion was successful otherwise clean up references and return null + if (!x || !y) { + Py_XDECREF(x); + Py_XDECREF(y); return nullptr; } - return PyTuple_Pack(2, x, y); + // Create tuple from x and y + auto t = PyTuple_Pack(2, x, y); + // Decrement references (the tuple now owns x an y) + Py_DECREF(x); + Py_DECREF(y); + return t; } // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The @@ -95,16 +100,18 @@ type is explicitly allowed. auto x = PySequence_GetItem(src.ptr(), 0); auto y = PySequence_GetItem(src.ptr(), 1); // Check if values are float or int (both are allowed with float as type hint) - if (!x || !(PyFloat_Check(x) || PyLong_Check(x))) { - return false; - } - if (!y || !(PyFloat_Check(y) || PyLong_Check(y))) { + if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y + || !(PyFloat_Check(y) || PyLong_Check(y))) { + Py_XDECREF(x); + Py_XDECREF(y); return false; } // value is a default constructed Point2D value.x = PyFloat_AsDouble(x); value.y = PyFloat_AsDouble(y); - if (PyErr_Occurred()) { + Py_DECREF(x); + Py_DECREF(y); + if ((value.x == -1.0 || value.y == -1.0) && PyErr_Occurred()) { PyErr_Clear(); return false; } @@ -138,4 +145,4 @@ type is explicitly allowed. Unfortunately, that loses the length information ``tuple[float, float]`` provides. One way of still providing some length information in type hints is using ``typing.Annotated``, e.g., ``Annotated[Sequence[float], 2]``, or further add libraries like - `annotated-types `. + `annotated-types `_. diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 5b2459b2f4..4ad73ac50a 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -33,16 +33,21 @@ struct type_caster { // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. static handle cast(const user_space::Point2D &number, return_value_policy, handle) { + // Convert x and y components to python float auto x = PyFloat_FromDouble(number.x); - if (!x) { - return nullptr; - } auto y = PyFloat_FromDouble(number.y); - if (!y) { - Py_DECREF(x); + // Check if conversion was successful otherwise clean up references and return null + if (!x || !y) { + Py_XDECREF(x); + Py_XDECREF(y); return nullptr; } - return PyTuple_Pack(2, x, y); + // Create tuple from x and y + auto t = PyTuple_Pack(2, x, y); + // Decrement references (the tuple now owns x an y) + Py_DECREF(x); + Py_DECREF(y); + return t; } // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The @@ -55,16 +60,18 @@ struct type_caster { auto x = PySequence_GetItem(src.ptr(), 0); auto y = PySequence_GetItem(src.ptr(), 1); // Check if values are float or int (both are allowed with float as type hint) - if (!x || !(PyFloat_Check(x) || PyLong_Check(x))) { - return false; - } - if (!y || !(PyFloat_Check(y) || PyLong_Check(y))) { + if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y + || !(PyFloat_Check(y) || PyLong_Check(y))) { + Py_XDECREF(x); + Py_XDECREF(y); return false; } // value is a default constructed Point2D value.x = PyFloat_AsDouble(x); value.y = PyFloat_AsDouble(y); - if (PyErr_Occurred()) { + Py_DECREF(x); + Py_DECREF(y); + if ((value.x == -1.0 || value.y == -1.0) && PyErr_Occurred()) { PyErr_Clear(); return false; } From 5ea6b690fcbfbbf2712f012ba11f01ea3aafe7c2 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 26 Nov 2024 17:08:07 +0100 Subject: [PATCH 21/32] Fixed clang-tidy issues. --- docs/advanced/cast/custom.rst | 10 +++++----- tests/test_docs_advanced_cast_custom.cpp | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 83aafbc6ec..27216e7dcf 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -74,8 +74,8 @@ type is explicitly allowed. // return_value_policy::reference_internal) and are often ignored by custom casters. static handle cast(const user_space::Point2D &number, return_value_policy, handle) { // Convert x and y components to python float - auto x = PyFloat_FromDouble(number.x); - auto y = PyFloat_FromDouble(number.y); + auto *x = PyFloat_FromDouble(number.x); + auto *y = PyFloat_FromDouble(number.y); // Check if conversion was successful otherwise clean up references and return null if (!x || !y) { Py_XDECREF(x); @@ -94,11 +94,11 @@ type is explicitly allowed. // second argument indicates whether implicit conversions should be allowed. bool load(handle src, bool) { // Check if handle is valid Sequence of length 2 - if (!src || !PySequence_Check(src.ptr()) || PySequence_Length(src.ptr()) != 2) { + if (!src || PySequence_Check(src.ptr()) == 0 || PySequence_Length(src.ptr()) != 2) { return false; } - auto x = PySequence_GetItem(src.ptr(), 0); - auto y = PySequence_GetItem(src.ptr(), 1); + auto *x = PySequence_GetItem(src.ptr(), 0); + auto *y = PySequence_GetItem(src.ptr(), 1); // Check if values are float or int (both are allowed with float as type hint) if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y || !(PyFloat_Check(y) || PyLong_Check(y))) { diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 4ad73ac50a..2225e38ae0 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -34,8 +34,8 @@ struct type_caster { // return_value_policy::reference_internal) and are often ignored by custom casters. static handle cast(const user_space::Point2D &number, return_value_policy, handle) { // Convert x and y components to python float - auto x = PyFloat_FromDouble(number.x); - auto y = PyFloat_FromDouble(number.y); + auto *x = PyFloat_FromDouble(number.x); + auto *y = PyFloat_FromDouble(number.y); // Check if conversion was successful otherwise clean up references and return null if (!x || !y) { Py_XDECREF(x); @@ -54,11 +54,11 @@ struct type_caster { // second argument indicates whether implicit conversions should be allowed. bool load(handle src, bool) { // Check if handle is valid Sequence of length 2 - if (!src || !PySequence_Check(src.ptr()) || PySequence_Length(src.ptr()) != 2) { + if (!src || PySequence_Check(src.ptr()) == 0 || PySequence_Length(src.ptr()) != 2) { return false; } - auto x = PySequence_GetItem(src.ptr(), 0); - auto y = PySequence_GetItem(src.ptr(), 1); + auto *x = PySequence_GetItem(src.ptr(), 0); + auto *y = PySequence_GetItem(src.ptr(), 1); // Check if values are float or int (both are allowed with float as type hint) if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y || !(PyFloat_Check(y) || PyLong_Check(y))) { From 34fe2731a631461c74c0f03247753b138313172e Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Tue, 26 Nov 2024 18:47:34 +0100 Subject: [PATCH 22/32] Fix for clang-tidy --- docs/advanced/cast/custom.rst | 2 +- tests/test_docs_advanced_cast_custom.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 27216e7dcf..51b14336fd 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -83,7 +83,7 @@ type is explicitly allowed. return nullptr; } // Create tuple from x and y - auto t = PyTuple_Pack(2, x, y); + auto *t = PyTuple_Pack(2, x, y); // Decrement references (the tuple now owns x an y) Py_DECREF(x); Py_DECREF(y); diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 2225e38ae0..070819d705 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -43,7 +43,7 @@ struct type_caster { return nullptr; } // Create tuple from x and y - auto t = PyTuple_Pack(2, x, y); + auto *t = PyTuple_Pack(2, x, y); // Decrement references (the tuple now owns x an y) Py_DECREF(x); Py_DECREF(y); From e7047036afe300f94870433f610c780598b7799c Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sat, 30 Nov 2024 12:24:35 +0100 Subject: [PATCH 23/32] Updated cast method to use pybind11 API rather than Python C API in custom caster example --- tests/test_docs_advanced_cast_custom.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 070819d705..783a1a7039 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -33,21 +33,7 @@ struct type_caster { // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. static handle cast(const user_space::Point2D &number, return_value_policy, handle) { - // Convert x and y components to python float - auto *x = PyFloat_FromDouble(number.x); - auto *y = PyFloat_FromDouble(number.y); - // Check if conversion was successful otherwise clean up references and return null - if (!x || !y) { - Py_XDECREF(x); - Py_XDECREF(y); - return nullptr; - } - // Create tuple from x and y - auto *t = PyTuple_Pack(2, x, y); - // Decrement references (the tuple now owns x an y) - Py_DECREF(x); - Py_DECREF(y); - return t; + return py::make_tuple(number.x, number.y).release(); } // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The From fa0a7b1e47fe155df77d527ad9dfc10879da6122 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 1 Dec 2024 14:34:10 +0100 Subject: [PATCH 24/32] Updated load to use pybind11 API rather than Python C API in custom caster example --- tests/test_docs_advanced_cast_custom.cpp | 34 +++++++++++------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 783a1a7039..03638fa486 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -32,35 +32,31 @@ struct type_caster { // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. - static handle cast(const user_space::Point2D &number, return_value_policy, handle) { + static handle + cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { return py::make_tuple(number.x, number.y).release(); } // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The // second argument indicates whether implicit conversions should be allowed. - bool load(handle src, bool) { - // Check if handle is valid Sequence of length 2 - if (!src || PySequence_Check(src.ptr()) == 0 || PySequence_Length(src.ptr()) != 2) { + bool load(handle src, bool /*convert*/) { + // Check if handle is a Sequence + if (!py::isinstance(src)) { return false; } - auto *x = PySequence_GetItem(src.ptr(), 0); - auto *y = PySequence_GetItem(src.ptr(), 1); - // Check if values are float or int (both are allowed with float as type hint) - if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y - || !(PyFloat_Check(y) || PyLong_Check(y))) { - Py_XDECREF(x); - Py_XDECREF(y); + auto seq = py::reinterpret_borrow(src); + // Check if exactly two values are in the Sequence + if (seq.size() != 2) { return false; } - // value is a default constructed Point2D - value.x = PyFloat_AsDouble(x); - value.y = PyFloat_AsDouble(y); - Py_DECREF(x); - Py_DECREF(y); - if ((value.x == -1.0 || value.y == -1.0) && PyErr_Occurred()) { - PyErr_Clear(); - return false; + // Check if each element is either a float or an int + for (auto item : seq) { + if (!py::isinstance(item) and !py::isinstance(item)) { + return false; + } } + value.x = seq[0].cast(); + value.y = seq[1].cast(); return true; } }; From 1fe101a628718dbcddeb7c6d5b85eb2c1c73585a Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 1 Dec 2024 14:55:03 +0100 Subject: [PATCH 25/32] Changed test of arg/return name to use pybind11 API instead of Python C API --- tests/test_pytypes.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index f2c39160f6..c5f3b82ddc 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -158,17 +158,14 @@ struct type_caster { static constexpr auto return_name = const_name("float"); static handle cast(const RealNumber &number, return_value_policy, handle) { - return PyFloat_FromDouble(number.value); + return py::float_(number.value).release(); } bool load(handle src, bool) { - if (!src) { + if (!py::isinstance(src) && !py::isinstance(src)) { return false; } - if (!PyFloat_Check(src.ptr()) && !PyLong_Check(src.ptr())) { - return false; - } - value.value = PyFloat_AsDouble(src.ptr()); + value.value = src.cast(); return true; } }; From 562153d41efe5a622494a6b7a1238dac4a4b3502 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 1 Dec 2024 15:24:41 +0100 Subject: [PATCH 26/32] Updated code in adcanced/cast example and improved documentation text --- docs/advanced/cast/custom.rst | 65 ++++++++++-------------- tests/test_docs_advanced_cast_custom.cpp | 6 ++- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 51b14336fd..42dc03807a 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -4,8 +4,9 @@ Custom type casters Some applications may prefer custom type casters that convert between existing Python types and C++ types, similar to the ``list`` ↔ ``std::vector`` and ``dict`` ↔ ``std::map`` conversions which are built into pybind11. -Implementing custom type casters is fairly advanced usage and requires -familiarity with the intricacies of the Python C API. +Implementing custom type casters is fairly advanced usage. +While it is recommended to use the pybind11 API as much as possible, more complex examples may +require familiarity with the intricacies of the Python C API. You can refer to the `Python/C API Reference Manual `_ for more information. @@ -64,57 +65,41 @@ type is explicitly allowed. PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); // `arg_name` and `return_name` may optionally be used to specify type hints separately for // arguments and return values. - // The signature of our identity function would then look like: - // `identity(Sequence[float]) -> tuple[float, float]` + // The signature of our negate function would then look like: + // `negate(Sequence[float]) -> tuple[float, float]` static constexpr auto arg_name = const_name("Sequence[float]"); static constexpr auto return_name = const_name("tuple[float, float]"); // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. - static handle cast(const user_space::Point2D &number, return_value_policy, handle) { - // Convert x and y components to python float - auto *x = PyFloat_FromDouble(number.x); - auto *y = PyFloat_FromDouble(number.y); - // Check if conversion was successful otherwise clean up references and return null - if (!x || !y) { - Py_XDECREF(x); - Py_XDECREF(y); - return nullptr; - } - // Create tuple from x and y - auto *t = PyTuple_Pack(2, x, y); - // Decrement references (the tuple now owns x an y) - Py_DECREF(x); - Py_DECREF(y); - return t; + // The return value should reflect the type hint specified by `return_name`. + static handle + cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { + return py::make_tuple(number.x, number.y).release(); } // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The // second argument indicates whether implicit conversions should be allowed. - bool load(handle src, bool) { - // Check if handle is valid Sequence of length 2 - if (!src || PySequence_Check(src.ptr()) == 0 || PySequence_Length(src.ptr()) != 2) { + // The accepted types should reflect the type hint specified by `arg_name`. + bool load(handle src, bool /*convert*/) { + // Check if handle is a Sequence + if (!py::isinstance(src)) { return false; } - auto *x = PySequence_GetItem(src.ptr(), 0); - auto *y = PySequence_GetItem(src.ptr(), 1); - // Check if values are float or int (both are allowed with float as type hint) - if (!x || !(PyFloat_Check(x) || PyLong_Check(x)) || !y - || !(PyFloat_Check(y) || PyLong_Check(y))) { - Py_XDECREF(x); - Py_XDECREF(y); + auto seq = py::reinterpret_borrow(src); + // Check if exactly two values are in the Sequence + if (seq.size() != 2) { return false; } - // value is a default constructed Point2D - value.x = PyFloat_AsDouble(x); - value.y = PyFloat_AsDouble(y); - Py_DECREF(x); - Py_DECREF(y); - if ((value.x == -1.0 || value.y == -1.0) && PyErr_Occurred()) { - PyErr_Clear(); - return false; + // Check if each element is either a float or an int + for (auto item : seq) { + if (!py::isinstance(item) and !py::isinstance(item)) { + return false; + } } + value.x = seq[0].cast(); + value.y = seq[1].cast(); return true; } }; @@ -131,6 +116,10 @@ type is explicitly allowed. that ``T`` is default-constructible (``value`` is first default constructed and then ``load()`` assigns to it). +.. note:: + For further information on the ``return_value_policy`` argument of ``cast`` refer to :ref:`return-value-policies`. + To learn about the ``convert`` argument of ``load`` see :ref:`non-converting-arguments`. + .. warning:: When using custom type casters, it's important to declare them consistently diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index 03638fa486..badb29bbe3 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -24,14 +24,15 @@ struct type_caster { PYBIND11_TYPE_CASTER(user_space::Point2D, const_name("tuple")); // `arg_name` and `return_name` may optionally be used to specify type hints separately for // arguments and return values. - // The signature of our identity function would then look like: - // `identity(Sequence[float]) -> tuple[float, float]` + // The signature of our negate function would then look like: + // `negate(Sequence[float]) -> tuple[float, float]` static constexpr auto arg_name = const_name("Sequence[float]"); static constexpr auto return_name = const_name("tuple[float, float]"); // C++ -> Python: convert `Point2D` to `tuple[float, float]`. The second and third arguments // are used to indicate the return value policy and parent object (for // return_value_policy::reference_internal) and are often ignored by custom casters. + // The return value should reflect the type hint specified by `return_name`. static handle cast(const user_space::Point2D &number, return_value_policy /*policy*/, handle /*parent*/) { return py::make_tuple(number.x, number.y).release(); @@ -39,6 +40,7 @@ struct type_caster { // Python -> C++: convert a `PyObject` into a `Point2D` and return false upon failure. The // second argument indicates whether implicit conversions should be allowed. + // The accepted types should reflect the type hint specified by `arg_name`. bool load(handle src, bool /*convert*/) { // Check if handle is a Sequence if (!py::isinstance(src)) { From eea98b2df53dd2dfffaacd0d080e041fc7f089de Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 1 Dec 2024 15:47:19 +0100 Subject: [PATCH 27/32] Fixed references in custom type caster docs --- docs/advanced/cast/custom.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 42dc03807a..4018392b74 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -117,8 +117,8 @@ type is explicitly allowed. and then ``load()`` assigns to it). .. note:: - For further information on the ``return_value_policy`` argument of ``cast`` refer to :ref:`return-value-policies`. - To learn about the ``convert`` argument of ``load`` see :ref:`non-converting-arguments`. + For further information on the ``return_value_policy`` argument of ``cast`` refer to :ref:`return_value_policies`. + To learn about the ``convert`` argument of ``load`` see :ref:`nonconverting_arguments`. .. warning:: From acb58b3894d6f22418d974708b11792dddef2189 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Sun, 1 Dec 2024 15:49:59 +0100 Subject: [PATCH 28/32] Fixed wrong logical and operator in test --- tests/test_docs_advanced_cast_custom.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_docs_advanced_cast_custom.cpp b/tests/test_docs_advanced_cast_custom.cpp index badb29bbe3..a6f8a212ef 100644 --- a/tests/test_docs_advanced_cast_custom.cpp +++ b/tests/test_docs_advanced_cast_custom.cpp @@ -53,7 +53,7 @@ struct type_caster { } // Check if each element is either a float or an int for (auto item : seq) { - if (!py::isinstance(item) and !py::isinstance(item)) { + if (!py::isinstance(item) && !py::isinstance(item)) { return false; } } From bab038d0c5ce908c65fd987d571111d6aa59ace1 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 2 Dec 2024 00:00:31 +0100 Subject: [PATCH 29/32] Fixed wrong logical operator in doc example --- docs/advanced/cast/custom.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index 4018392b74..bc3325d6fb 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -94,7 +94,7 @@ type is explicitly allowed. } // Check if each element is either a float or an int for (auto item : seq) { - if (!py::isinstance(item) and !py::isinstance(item)) { + if (!py::isinstance(item) && !py::isinstance(item)) { return false; } } From 3a78cb4ad581aa069fd6eea199c1cd4a8ce69a73 Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Mon, 2 Dec 2024 07:41:31 +0100 Subject: [PATCH 30/32] Added comment to test about `float` vs `float | int` --- tests/test_pytypes.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/test_pytypes.cpp b/tests/test_pytypes.cpp index c5f3b82ddc..1764ccda02 100644 --- a/tests/test_pytypes.cpp +++ b/tests/test_pytypes.cpp @@ -140,9 +140,12 @@ typedef py::typing::TypeVar<"V"> TypeVarV; // Custom type for testing arg_name/return_name type hints // RealNumber: -// in arguments -> float | int, -// in return -> float -// fallback -> complex (just for testing, not really useful here) +// * in arguments -> float | int +// * in return -> float +// * fallback -> complex +// The choice of types is not really useful, but just made different for testing purposes. +// According to `PEP 484 – Type Hints` annotating with `float` also allows `int`, +// so using `float | int` could be replaced by just `float`. struct RealNumber { double value; From 6ea4704d538a9744568d92be7dead2f384626c3c Mon Sep 17 00:00:00 2001 From: Tim Ohliger Date: Thu, 5 Dec 2024 12:29:11 +0100 Subject: [PATCH 31/32] Updated std::filesystem::path docs in cast/overview section --- docs/advanced/cast/overview.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/advanced/cast/overview.rst b/docs/advanced/cast/overview.rst index 011bd4c7a3..d5a34ef942 100644 --- a/docs/advanced/cast/overview.rst +++ b/docs/advanced/cast/overview.rst @@ -151,7 +151,7 @@ as arguments and return values, refer to the section on binding :ref:`classes`. +------------------------------------+---------------------------+-----------------------------------+ | ``std::variant<...>`` | Type-safe union (C++17) | :file:`pybind11/stl.h` | +------------------------------------+---------------------------+-----------------------------------+ -| ``std::filesystem::path`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | +| ``std::filesystem::path`` | STL path (C++17) [#]_ | :file:`pybind11/stl/filesystem.h` | +------------------------------------+---------------------------+-----------------------------------+ | ``std::function<...>`` | STL polymorphic function | :file:`pybind11/functional.h` | +------------------------------------+---------------------------+-----------------------------------+ @@ -167,4 +167,4 @@ as arguments and return values, refer to the section on binding :ref:`classes`. +------------------------------------+---------------------------+-----------------------------------+ .. [#] ``std::filesystem::path`` is converted to ``pathlib.Path`` and - ``os.PathLike`` is converted to ``std::filesystem::path``. + can be loaded from ``os.PathLike``, ``str``, and ``bytes``. From aa21ab5294a406e602e468c14e0b3dd53c0c1341 Mon Sep 17 00:00:00 2001 From: "Ralf W. Grosse-Kunstleve" Date: Sun, 8 Dec 2024 10:20:55 -0800 Subject: [PATCH 32/32] Remove one stray dot. --- docs/advanced/cast/custom.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/cast/custom.rst b/docs/advanced/cast/custom.rst index bc3325d6fb..065d09a6dd 100644 --- a/docs/advanced/cast/custom.rst +++ b/docs/advanced/cast/custom.rst @@ -124,7 +124,7 @@ type is explicitly allowed. When using custom type casters, it's important to declare them consistently in every compilation unit of the Python extension module to satisfy the C++ One Definition Rule - (`ODR `_).. Otherwise, + (`ODR `_). Otherwise, undefined behavior can ensue. .. note::