[WIP] Interoperability with other Python binding frameworks

.clang-tidy

@@ -78,3 +78,4 @@ CheckOptions:
   value:           true
 
 HeaderFilterRegex: 'pybind11/.*h'
+ExcludeHeaderFilterRegex: 'pybind11/detail/pymetabind.h'

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ ci:
   autoupdate_schedule: monthly
 
 # third-party content
-exclude: ^tools/JoinPaths.cmake$
+exclude: ^(tools/JoinPaths.cmake|include/pybind11/detail/pymetabind.h)$
 
 repos:
 

CMakeLists.txt

@@ -186,11 +186,13 @@ set(PYBIND11_HEADERS
     include/pybind11/detail/descr.h
     include/pybind11/detail/dynamic_raw_ptr_cast_if_possible.h
     include/pybind11/detail/exception_translation.h
+    include/pybind11/detail/foreign.h
     include/pybind11/detail/function_record_pyobject.h
     include/pybind11/detail/init.h
     include/pybind11/detail/internals.h
     include/pybind11/detail/native_enum_data.h
     include/pybind11/detail/pybind11_namespace_macros.h
+    include/pybind11/detail/pymetabind.h
     include/pybind11/detail/struct_smart_holder.h
     include/pybind11/detail/type_caster_base.h
     include/pybind11/detail/typeid.h

include/pybind11/cast.h

@@ -862,6 +862,71 @@ struct holder_helper {
     static auto get(const T &p) -> decltype(p.get()) { return p.get(); }
 };
 
+struct holder_caster_foreign_helpers {
+    struct py_deleter {
+        void operator()(const void *) const noexcept {
+            // Don't run the deleter if the interpreter has been shut down
+            if (Py_IsInitialized() == 0) {
+                return;
+            }
+            gil_scoped_acquire guard;
+            Py_DECREF(o);
+        }
+
+        PyObject *o;
+    };
+
+    template <typename base, typename type>
+    static bool try_shared_from_this(std::enable_shared_from_this<base> *value,
+                                     std::shared_ptr<type> *holder_out) {
+        // object derives from enable_shared_from_this;
+        // try to reuse an existing shared_ptr if one is known
+        if (auto existing = try_get_shared_from_this(value)) {
+            *holder_out = std::static_pointer_cast<type>(existing);
+            return true;
+        }
+        return false;
+    }
+
+    template <typename type>
+    static bool try_shared_from_this(void *, std::shared_ptr<type> *) {
+        return false;
+    }
+
+    template <typename type>
+    static bool set_foreign_holder(handle src, type *value, std::shared_ptr<type> *holder_out) {
+        // We only support using std::shared_ptr<T> for foreign T, and
+        // it's done by creating a new shared_ptr control block that
+        // owns a reference to the original Python object.
+        if (value == nullptr) {
+            *holder_out = {};
+            return true;
+        }
+        if (try_shared_from_this(value, holder_out)) {
+            return true;
+        }
+        *holder_out = std::shared_ptr<type>(value, py_deleter{src.inc_ref().ptr()});
+        return true;
+    }
+
+    template <typename type>
+    static bool
+    set_foreign_holder(handle src, type *value, std::shared_ptr<const type> *holder_out) {
+        std::shared_ptr<type> holder_mut;
+        if (set_foreign_holder(src, value, &holder_mut)) {
+            *holder_out = holder_mut;
+            return true;
+        }
+        return false;
+    }
+
+    template <typename type>
+    static bool set_foreign_holder(handle, type *, ...) {
+        throw cast_error("Unable to cast foreign type to held instance -- "
+                         "only std::shared_ptr<T> is supported in this case");
+    }
+};
+
 // SMART_HOLDER_BAKEIN_FOLLOW_ON: Rewrite comment, with reference to shared_ptr specialization.
 /// Type caster for holder types like std::shared_ptr, etc.
 /// The SFINAE hook is provided to help work around the current lack of support
@@ -906,6 +971,10 @@ struct copyable_holder_caster : public type_caster_base<type> {
         }
     }
 
+    bool set_foreign_holder(handle src) {
+        return holder_caster_foreign_helpers::set_foreign_holder(src, (type *) value, &holder);
+    }
+
     void load_value(value_and_holder &&v_h) {
         if (v_h.holder_constructed()) {
             value = v_h.value_ptr();
@@ -976,22 +1045,22 @@ struct copyable_holder_caster<
     }
 
     explicit operator std::shared_ptr<type> *() {
-        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+        if (sh_load_helper.was_populated) {
             pybind11_fail("Passing `std::shared_ptr<T> *` from Python to C++ is not supported "
                           "(inherently unsafe).");
         }
         return std::addressof(shared_ptr_storage);
     }
 
     explicit operator std::shared_ptr<type> &() {
-        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+        if (sh_load_helper.was_populated) {
             shared_ptr_storage = sh_load_helper.load_as_shared_ptr(typeinfo, value);
         }
         return shared_ptr_storage;
     }
 
     std::weak_ptr<type> potentially_slicing_weak_ptr() {
-        if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
+        if (sh_load_helper.was_populated) {
             // Reusing shared_ptr code to minimize code complexity.
             shared_ptr_storage
                 = sh_load_helper.load_as_shared_ptr(typeinfo,
@@ -1005,15 +1074,12 @@ struct copyable_holder_caster<
     static handle
     cast(const std::shared_ptr<type> &src, return_value_policy policy, handle parent) {
         const auto *ptr = src.get();
-        auto st = type_caster_base<type>::src_and_type(ptr);
-        if (st.second == nullptr) {
-            return handle(); // no type info: error will be set already
-        }
-        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
+        typename type_caster_base<type>::cast_sources srcs{ptr};
+        if (srcs.creates_smart_holder()) {
             return smart_holder_type_caster_support::smart_holder_from_shared_ptr(
-                src, policy, parent, st);
+                src, policy, parent, srcs.result);
         }
-        return type_caster_base<type>::cast_holder(ptr, &src);
+        return type_caster_base<type>::cast_holder(srcs, &src);
     }
 
     // This function will succeed even if the `responsible_parent` does not own the
@@ -1040,6 +1106,11 @@ struct copyable_holder_caster<
         }
     }
 
+    bool set_foreign_holder(handle src) {
+        return holder_caster_foreign_helpers::set_foreign_holder(
+            src, (type *) value, &shared_ptr_storage);
+    }
+
     void load_value(value_and_holder &&v_h) {
         if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
@@ -1077,6 +1148,7 @@ struct copyable_holder_caster<
                 value = cast.second(sub_caster.value);
                 if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
                     sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
+                    sh_load_helper.was_populated = true;
                 } else {
                     shared_ptr_storage
                         = std::shared_ptr<type>(sub_caster.shared_ptr_storage, (type *) value);
@@ -1183,21 +1255,12 @@ struct move_only_holder_caster<
     static handle
     cast(std::unique_ptr<type, deleter> &&src, return_value_policy policy, handle parent) {
         auto *ptr = src.get();
-        auto st = type_caster_base<type>::src_and_type(ptr);
-        if (st.second == nullptr) {
-            return handle(); // no type info: error will be set already
-        }
-        if (st.second->holder_enum_v == detail::holder_enum_t::smart_holder) {
+        typename type_caster_base<type>::cast_sources srcs{ptr};
+        if (srcs.creates_smart_holder()) {
             return smart_holder_type_caster_support::smart_holder_from_unique_ptr(
-                std::move(src), policy, parent, st);
+                std::move(src), policy, parent, srcs.result);
         }
-        return type_caster_generic::cast(st.first,
-                                         return_value_policy::take_ownership,
-                                         {},
-                                         st.second,
-                                         nullptr,
-                                         nullptr,
-                                         std::addressof(src));
+        return type_caster_base<type>::cast_holder(srcs, &src);
     }
 
     static handle
@@ -1223,6 +1286,12 @@ struct move_only_holder_caster<
         return false;
     }
 
+    bool set_foreign_holder(handle) {
+        throw cast_error("Foreign types cannot be converted to std::unique_ptr "
+                         "because we don't know how to make them relinquish "
+                         "ownership");
+    }
+
     void load_value(value_and_holder &&v_h) {
         if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
             sh_load_helper.loaded_v_h = v_h;
@@ -1281,6 +1350,7 @@ struct move_only_holder_caster<
                 value = cast.second(sub_caster.value);
                 if (typeinfo->holder_enum_v == detail::holder_enum_t::smart_holder) {
                     sh_load_helper.loaded_v_h = sub_caster.sh_load_helper.loaded_v_h;
+                    sh_load_helper.was_populated = true;
                 } else {
                     pybind11_fail("Expected to be UNREACHABLE: " __FILE__
                                   ":" PYBIND11_TOSTRING(__LINE__));
@@ -2338,11 +2408,11 @@ object object_api<Derived>::call(Args &&...args) const {
 PYBIND11_NAMESPACE_END(detail)
 
 template <typename T>
-handle type::handle_of() {
+handle type::handle_of(bool foreign_ok) {
     static_assert(std::is_base_of<detail::type_caster_generic, detail::make_caster<T>>::value,
-                  "py::type::of<T> only supports the case where T is a registered C++ types.");
+                  "py::type::of<T> only supports the case where T is a registered C++ type.");
 
-    return detail::get_type_handle(typeid(T), true);
+    return detail::get_type_handle(typeid(T), true, foreign_ok);
 }
 
 #define PYBIND11_MAKE_OPAQUE(...)                                                                 \

include/pybind11/detail/class.h

@@ -226,6 +226,7 @@ extern "C" inline void pybind11_meta_dealloc(PyObject *obj) {
             } else {
                 internals.registered_types_cpp.erase(tindex);
             }
+            get_foreign_internals().copy_move_ctors.erase(tindex);
             internals.registered_types_py.erase(tinfo->type);
 
             // Actually just `std::erase_if`, but that's only available in C++20

include/pybind11/detail/common.h

@@ -214,6 +214,12 @@
 #    define PYBIND11_SIMPLE_GIL_MANAGEMENT
 #endif
 
+#if defined(_MSC_VER)
+#    define PYBIND11_COMPAT_STRDUP _strdup
+#else
+#    define PYBIND11_COMPAT_STRDUP strdup
+#endif
+
 #include <cstddef>
 #include <cstring>
 #include <exception>
@@ -348,6 +354,7 @@
 #define PYBIND11_ENSURE_INTERNALS_READY                                                           \
     {                                                                                             \
         pybind11::detail::get_internals_pp_manager().unref();                                     \
+        pybind11::detail::get_foreign_internals_pp_manager().unref();                             \
         pybind11::detail::get_internals();                                                        \
     }