Merge branch 'master' into expand-float-strict

Author: InvincibleRMC

include/pybind11/cast.h

@@ -1469,21 +1469,24 @@ template <>
 struct handle_type_name<weakref> {
     static constexpr auto name = const_name("weakref.ReferenceType");
 };
+// args/Args/kwargs/KWArgs have name as well as typehint included
 template <>
 struct handle_type_name<args> {
-    static constexpr auto name = const_name("*args");
+    static constexpr auto name = io_name("*args", "tuple");
 };
 template <typename T>
 struct handle_type_name<Args<T>> {
-    static constexpr auto name = const_name("*args: ") + make_caster<T>::name;
+    static constexpr auto name
+        = io_name("*args: ", "tuple[") + make_caster<T>::name + io_name("", ", ...]");
 };
 template <>
 struct handle_type_name<kwargs> {
-    static constexpr auto name = const_name("**kwargs");
+    static constexpr auto name = io_name("**kwargs", "dict[str, typing.Any]");
 };
 template <typename T>
 struct handle_type_name<KWArgs<T>> {
-    static constexpr auto name = const_name("**kwargs: ") + make_caster<T>::name;
+    static constexpr auto name
+        = io_name("**kwargs: ", "dict[str, ") + make_caster<T>::name + io_name("", "]");
 };
 template <>
 struct handle_type_name<obj_attr_accessor> {
@@ -1909,13 +1912,20 @@ inline cast_error cast_error_unable_to_convert_call_arg(const std::string &name,
 }
 #endif
 
+namespace typing {
+template <typename... Types>
+class Tuple : public tuple {
+    using tuple::tuple;
+};
+} // namespace typing
+
 template <return_value_policy policy = return_value_policy::automatic_reference>
-tuple make_tuple() {
+typing::Tuple<> make_tuple() {
     return tuple(0);
 }
 
 template <return_value_policy policy = return_value_policy::automatic_reference, typename... Args>
-tuple make_tuple(Args &&...args_) {
+typing::Tuple<Args...> make_tuple(Args &&...args_) {
     constexpr size_t size = sizeof...(Args);
     std::array<object, size> args{{reinterpret_steal<object>(
         detail::make_caster<Args>::cast(std::forward<Args>(args_), policy, nullptr))...}};
@@ -1934,7 +1944,12 @@ tuple make_tuple(Args &&...args_) {
     for (auto &arg_value : args) {
         PyTuple_SET_ITEM(result.ptr(), counter++, arg_value.release().ptr());
     }
+    PYBIND11_WARNING_PUSH
+#ifdef PYBIND11_DETECTED_CLANG_WITH_MISLEADING_CALL_STD_MOVE_EXPLICITLY_WARNING
+    PYBIND11_WARNING_DISABLE_CLANG("-Wreturn-std-move")
+#endif
     return result;
+    PYBIND11_WARNING_POP
 }
 
 /// \ingroup annotations

include/pybind11/detail/common.h

@@ -103,6 +103,10 @@
 #    define PYBIND11_DTOR_CONSTEXPR
 #endif
 
+#if defined(PYBIND11_CPP20) && defined(__has_include) && __has_include(<barrier>)
+#    define PYBIND11_HAS_STD_BARRIER 1
+#endif
+
 // Compiler version assertions
 #if defined(__INTEL_COMPILER)
 #    if __INTEL_COMPILER < 1800

include/pybind11/detail/init.h

@@ -501,9 +501,15 @@ template <typename Get,
           typename NewInstance,
           typename ArgState>
 struct pickle_factory<Get, Set, RetState(Self), NewInstance(ArgState)> {
-    static_assert(std::is_same<intrinsic_t<RetState>, intrinsic_t<ArgState>>::value,
-                  "The type returned by `__getstate__` must be the same "
-                  "as the argument accepted by `__setstate__`");
+    using Ret = intrinsic_t<RetState>;
+    using Arg = intrinsic_t<ArgState>;
+
+    // Subclasses are now allowed for support between type hint and generic versions of types
+    // (e.g.) typing::List <--> list
+    static_assert(std::is_same<Ret, Arg>::value || std::is_base_of<Ret, Arg>::value
+                      || std::is_base_of<Arg, Ret>::value,
+                  "The type returned by `__getstate__` must be the same or subclass of the "
+                  "argument accepted by `__setstate__`");
 
     remove_reference_t<Get> get;
     remove_reference_t<Set> set;

include/pybind11/gil.h

@@ -120,7 +120,11 @@ class gil_scoped_acquire {
                 pybind11_fail("scoped_acquire::dec_ref(): internal error!");
             }
 #    endif
+            // Make sure that PyThreadState_Clear is not recursively called by finalizers.
+            // See issue #5827
+            ++tstate->gilstate_counter;
             PyThreadState_Clear(tstate);
+            --tstate->gilstate_counter;
             if (active) {
                 PyThreadState_DeleteCurrent();
             }

include/pybind11/pybind11.h

@@ -120,7 +120,8 @@ inline std::string generate_function_signature(const char *type_caster_name_fiel
         const auto c = *pc;
         if (c == '{') {
             // Write arg name for everything except *args and **kwargs.
-            is_starred = *(pc + 1) == '*';
+            // Detect {@*args...} or {@**kwargs...}
+            is_starred = *(pc + 1) == '@' && *(pc + 2) == '*';
             if (is_starred) {
                 continue;
             }

include/pybind11/typing.h

@@ -34,10 +34,7 @@ PYBIND11_NAMESPACE_BEGIN(typing)
     There is no additional enforcement of types at runtime.
 */
 
-template <typename... Types>
-class Tuple : public tuple {
-    using tuple::tuple;
-};
+// Tuple type hint defined in cast.h for use in py::make_tuple to avoid circular includes
 
 template <typename K, typename V>
 class Dict : public dict {

tests/test_factory_constructors.cpp

@@ -376,10 +376,14 @@ TEST_SUBMODULE(factory_constructors, m) {
             py::print("noisy placement new");
             return p;
         }
-        static void operator delete(void *p, size_t) {
+        static void operator delete(void *p) noexcept {
             py::print("noisy delete");
             ::operator delete(p);
         }
+        static void operator delete(void *p, size_t) {
+            py::print("noisy delete size");
+            ::operator delete(p);
+        }
         static void operator delete(void *, void *) { py::print("noisy placement delete"); }
     };
 

tests/test_kwargs_and_defaults.py

@@ -34,11 +34,12 @@ def test_function_signatures(doc):
     )
     assert doc(m.args_function) == "args_function(*args) -> tuple"
     assert (
-        doc(m.args_kwargs_function) == "args_kwargs_function(*args, **kwargs) -> tuple"
+        doc(m.args_kwargs_function)
+        == "args_kwargs_function(*args, **kwargs) -> tuple[tuple, dict[str, typing.Any]]"
     )
     assert (
         doc(m.args_kwargs_subclass_function)
-        == "args_kwargs_subclass_function(*args: str, **kwargs: str) -> tuple"
+        == "args_kwargs_subclass_function(*args: str, **kwargs: str) -> tuple[tuple[str, ...], dict[str, str]]"
     )
     assert (
         doc(m.KWClass.foo0)
@@ -138,7 +139,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple
+            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple[int, float, tuple]
 
         Invoked with: 1
     """
@@ -149,7 +150,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args(): incompatible function arguments. The following argument types are supported:
-            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple
+            1. (arg0: typing.SupportsInt | typing.SupportsIndex, arg1: typing.SupportsFloat | typing.SupportsIndex, *args) -> tuple[int, float, tuple]
 
         Invoked with:
     """
@@ -183,7 +184,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple[int, float, tuple, dict[str, typing.Any]]
 
         Invoked with: 1; kwargs: i=1
     """
@@ -194,7 +195,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         mixed_plus_args_kwargs_defaults(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, **kwargs) -> tuple[int, float, tuple, dict[str, typing.Any]]
 
         Invoked with: 1, 2; kwargs: j=1
     """
@@ -211,7 +212,7 @@ def test_mixed_args_and_kwargs(msg):
         msg(excinfo.value)
         == """
         args_kwonly(): incompatible function arguments. The following argument types are supported:
-            1. (i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex) -> tuple
+            1. (i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, float, tuple, int]
 
         Invoked with: 2, 2.5, 22
     """
@@ -233,12 +234,12 @@ def test_mixed_args_and_kwargs(msg):
     )
     assert (
         m.args_kwonly_kwargs.__doc__
-        == "args_kwonly_kwargs(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex, **kwargs) -> tuple\n"
+        == "args_kwonly_kwargs(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsFloat | typing.SupportsIndex, *args, z: typing.SupportsInt | typing.SupportsIndex, **kwargs) -> tuple[int, float, tuple, int, dict[str, typing.Any]]\n"
     )
 
     assert (
         m.args_kwonly_kwargs_defaults.__doc__
-        == "args_kwonly_kwargs_defaults(i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple\n"
+        == "args_kwonly_kwargs_defaults(i: typing.SupportsInt | typing.SupportsIndex = 1, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple[int, float, tuple, int, dict[str, typing.Any]]\n"
     )
     assert m.args_kwonly_kwargs_defaults() == (1, 3.14159, (), 42, {})
     assert m.args_kwonly_kwargs_defaults(2) == (2, 3.14159, (), 42, {})
@@ -344,7 +345,7 @@ def test_positional_only_args():
     # Mix it with args and kwargs:
     assert (
         m.args_kwonly_full_monty.__doc__
-        == "args_kwonly_full_monty(arg0: typing.SupportsInt | typing.SupportsIndex = 1, arg1: typing.SupportsInt | typing.SupportsIndex = 2, /, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple\n"
+        == "args_kwonly_full_monty(arg0: typing.SupportsInt | typing.SupportsIndex = 1, arg1: typing.SupportsInt | typing.SupportsIndex = 2, /, j: typing.SupportsFloat | typing.SupportsIndex = 3.14159, *args, z: typing.SupportsInt | typing.SupportsIndex = 42, **kwargs) -> tuple[int, int, float, tuple, int, dict[str, typing.Any]]\n"
     )
     assert m.args_kwonly_full_monty() == (1, 2, 3.14159, (), 42, {})
     assert m.args_kwonly_full_monty(8) == (8, 2, 3.14159, (), 42, {})
@@ -394,23 +395,23 @@ def test_positional_only_args():
 def test_signatures():
     assert (
         m.kw_only_all.__doc__
-        == "kw_only_all(*, i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "kw_only_all(*, i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.kw_only_mixed.__doc__
-        == "kw_only_mixed(i: typing.SupportsInt | typing.SupportsIndex, *, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "kw_only_mixed(i: typing.SupportsInt | typing.SupportsIndex, *, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.pos_only_all.__doc__
-        == "pos_only_all(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex, /) -> tuple\n"
+        == "pos_only_all(i: typing.SupportsInt | typing.SupportsIndex, j: typing.SupportsInt | typing.SupportsIndex, /) -> tuple[int, int]\n"
     )
     assert (
         m.pos_only_mix.__doc__
-        == "pos_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "pos_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int]\n"
     )
     assert (
         m.pos_kw_only_mix.__doc__
-        == "pos_kw_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex, *, k: typing.SupportsInt | typing.SupportsIndex) -> tuple\n"
+        == "pos_kw_only_mix(i: typing.SupportsInt | typing.SupportsIndex, /, j: typing.SupportsInt | typing.SupportsIndex, *, k: typing.SupportsInt | typing.SupportsIndex) -> tuple[int, int, int]\n"
     )
 
 

tests/test_scoped_critical_section.cpp

@@ -7,8 +7,7 @@
 #include <thread>
 #include <utility>
 
-#if defined(PYBIND11_CPP20) && defined(__has_include) && __has_include(<barrier>)
-#    define PYBIND11_HAS_BARRIER 1
+#if defined(PYBIND11_HAS_STD_BARRIER)
 #    include <barrier>
 #endif
 
@@ -39,7 +38,7 @@ class BoolWrapper {
     std::atomic_bool value_{false};
 };
 
-#if defined(PYBIND11_HAS_BARRIER)
+#if defined(PYBIND11_HAS_STD_BARRIER)
 
 // Modifying the C/C++ members of a Python object from multiple threads requires a critical section
 // to ensure thread safety and data integrity.
@@ -259,7 +258,7 @@ TEST_SUBMODULE(scoped_critical_section, m) {
     (void) BoolWrapperHandle.ptr(); // suppress unused variable warning
 
     m.attr("has_barrier") =
-#ifdef PYBIND11_HAS_BARRIER
+#ifdef PYBIND11_HAS_STD_BARRIER
         true;
 #else
         false;

tests/test_thread.cpp

@@ -15,6 +15,10 @@
 #include <chrono>
 #include <thread>
 
+#if defined(PYBIND11_HAS_STD_BARRIER)
+#    include <barrier>
+#endif
+
 namespace py = pybind11;
 
 namespace {
@@ -34,7 +38,6 @@ EmptyStruct SharedInstance;
 } // namespace
 
 TEST_SUBMODULE(thread, m) {
-
     py::class_<IntStruct>(m, "IntStruct").def(py::init([](const int i) { return IntStruct(i); }));
 
     // implicitly_convertible uses loader_life_support when an implicit
@@ -67,6 +70,39 @@ TEST_SUBMODULE(thread, m) {
     py::class_<EmptyStruct>(m, "EmptyStruct")
         .def_readonly_static("SharedInstance", &SharedInstance);
 
+#if defined(PYBIND11_HAS_STD_BARRIER)
+    // In the free-threaded build, during PyThreadState_Clear, removing the thread from the biased
+    // reference counting table may call destructors. Make sure that it doesn't crash.
+    m.def("test_pythread_state_clear_destructor", [](py::type cls) {
+        py::handle obj;
+
+        std::barrier barrier{2};
+        std::thread thread1{[&]() {
+            py::gil_scoped_acquire gil;
+            obj = cls().release();
+            barrier.arrive_and_wait();
+        }};
+        std::thread thread2{[&]() {
+            py::gil_scoped_acquire gil;
+            barrier.arrive_and_wait();
+            // ob_ref_shared becomes negative; transition to the queued state
+            obj.dec_ref();
+        }};
+
+        // jthread is not supported by Apple Clang
+        thread1.join();
+        thread2.join();
+    });
+#endif
+
+    m.attr("defined_PYBIND11_HAS_STD_BARRIER") =
+#ifdef PYBIND11_HAS_STD_BARRIER
+        true;
+#else
+        false;
+#endif
+    m.def("acquire_gil", []() { py::gil_scoped_acquire gil_acquired; });
+
     // NOTE: std::string_view also uses loader_life_support to ensure that
     // the string contents remain alive, but that's a C++ 17 feature.
 }