switch to heap types

Author: MilesCranmer

src/C/consts.jl

@@ -324,6 +324,32 @@ end
 
 const PyTypePtr = Ptr{PyTypeObject}
 
+@kwdef struct PyType_Slot
+    slot::Cint = 0
+    pfunc::Ptr{Cvoid} = C_NULL
+end
+
+@kwdef struct PyType_Spec
+    name::Cstring = C_NULL
+    basicsize::Cint = 0
+    itemsize::Cint = 0
+    flags::Cuint = 0
+    slots::Ptr{PyType_Slot} = C_NULL
+end
+
+# These numbers are part of the CPython stable ABI and
+# are guaranteed to be the same.
+# https://raw.githubusercontent.com/python/cpython/main/Include/typeslots.h
+const Py_bf_getbuffer = 1
+const Py_bf_releasebuffer = 2
+const Py_tp_alloc = 47
+const Py_tp_dealloc = 52
+const Py_tp_methods = 64
+const Py_tp_new = 65
+const Py_tp_members = 72
+const Py_tp_getset = 73
+const Py_tp_free = 74
+
 @kwdef struct PySimpleObject{T}
     ob_base::PyObject = PyObject()
     value::T

src/C/extras.jl

@@ -13,32 +13,32 @@ PyType_IsSubtypeFast(t, f::Integer) =
 PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
 
 PyType_CheckBuffer(t) = Base.GC.@preserve t begin
-    p = UnsafePtr{PyTypeObject}(asptr(t)).as_buffer[]
-    return p != C_NULL && p.get[!] != C_NULL
+    p = PyType_GetSlot(asptr(t), Py_bf_getbuffer)
+    return p != C_NULL
 end
 
 PyObject_CheckBuffer(o) = Base.GC.@preserve o PyType_CheckBuffer(Py_Type(asptr(o)))
 
 PyObject_GetBuffer(_o, b, flags) = Base.GC.@preserve _o begin
     o = asptr(_o)
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if p == C_NULL || p.get[!] == C_NULL
+    getbuf = PyType_GetSlot(Py_Type(o), Py_bf_getbuffer)
+    if getbuf == C_NULL
         PyErr_SetString(
             POINTERS.PyExc_TypeError,
             "a bytes-like object is required, not '$(String(UnsafePtr{PyTypeObject}(Py_Type(o)).name[]))'",
         )
         return Cint(-1)
     end
-    return ccall(p.get[!], Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
+    return ccall(getbuf, Cint, (PyPtr, Ptr{Py_buffer}, Cint), o, b, flags)
 end
 
 PyBuffer_Release(_b) = begin
     b = UnsafePtr(Base.unsafe_convert(Ptr{Py_buffer}, _b))
     o = b.obj[]
     o == C_NULL && return
-    p = UnsafePtr{PyTypeObject}(Py_Type(o)).as_buffer[]
-    if (p != C_NULL && p.release[!] != C_NULL)
-        ccall(p.release[!], Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
+    releasebuf = PyType_GetSlot(Py_Type(o), Py_bf_releasebuffer)
+    if releasebuf != C_NULL
+        ccall(releasebuf, Cvoid, (PyPtr, Ptr{Py_buffer}), o, b)
     end
     b.obj[] = C_NULL
     Py_DecRef(o)

src/C/pointers.jl

@@ -77,6 +77,9 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyType_IsSubtype => (PyPtr, PyPtr) => Cint,
     :PyType_Ready => (PyPtr,) => Cint,
     :PyType_GenericNew => (PyPtr, PyPtr, PyPtr) => PyPtr,
+    :PyType_FromSpec => (Ptr{PyType_Spec},) => PyPtr,
+    :PyType_FromSpecWithBases => (Ptr{PyType_Spec}, PyPtr) => PyPtr,
+    :PyType_GetSlot => (PyPtr, Cint) => Ptr{Cvoid},
     # MAPPING
     :PyMapping_HasKeyString => (PyPtr, Ptr{Cchar}) => Cint,
     :PyMapping_SetItemString => (PyPtr, Ptr{Cchar}, PyPtr) => Cint,

src/JlWrap/C.jl

@@ -9,7 +9,6 @@ using Serialization: serialize, deserialize
 @kwdef struct PyJuliaValueObject
     ob_base::C.PyObject = C.PyObject()
     value::Int = 0
-    weaklist::C.PyPtr = C_NULL
 end
 
 const PyJuliaBase_Type = Ref(C.PyNULL)
@@ -21,9 +20,9 @@ const PYJLVALUES = []
 const PYJLFREEVALUES = Int[]
 
 function _pyjl_new(t::C.PyPtr, ::C.PyPtr, ::C.PyPtr)
-    o = ccall(UnsafePtr{C.PyTypeObject}(t).alloc[!], C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
+    allocptr = C.PyType_GetSlot(t, C.Py_tp_alloc)
+    o = ccall(allocptr, C.PyPtr, (C.PyPtr, C.Py_ssize_t), t, 0)
     o == C.PyNULL && return C.PyNULL
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[] = C.PyNULL
     UnsafePtr{PyJuliaValueObject}(o).value[] = 0
     return o
 end
@@ -34,8 +33,8 @@ function _pyjl_dealloc(o::C.PyPtr)
         PYJLVALUES[idx] = nothing
         push!(PYJLFREEVALUES, idx)
     end
-    UnsafePtr{PyJuliaValueObject}(o).weaklist[!] == C.PyNULL || C.PyObject_ClearWeakRefs(o)
-    ccall(UnsafePtr{C.PyTypeObject}(C.Py_Type(o)).free[!], Cvoid, (C.PyPtr,), o)
+    freeptr = C.PyType_GetSlot(C.Py_Type(o), C.Py_tp_free)
+    ccall(freeptr, Cvoid, (C.PyPtr,), o)
     nothing
 end
 
@@ -269,14 +268,12 @@ function _pyjl_deserialize(t::C.PyPtr, v::C.PyPtr)
 end
 
 const _pyjlbase_name = "juliacall.ValueBase"
-const _pyjlbase_type = fill(C.PyTypeObject())
 const _pyjlbase_isnull_name = "_jl_isnull"
 const _pyjlbase_callmethod_name = "_jl_callmethod"
 const _pyjlbase_reduce_name = "__reduce__"
 const _pyjlbase_serialize_name = "_jl_serialize"
 const _pyjlbase_deserialize_name = "_jl_deserialize"
 const _pyjlbase_methods = Vector{C.PyMethodDef}()
-const _pyjlbase_as_buffer = fill(C.PyBufferProcs())
 
 function init_c()
     empty!(_pyjlbase_methods)
@@ -309,28 +306,37 @@ function init_c()
         ),
         C.PyMethodDef(),
     )
-    _pyjlbase_as_buffer[] = C.PyBufferProcs(
-        get = @cfunction(_pyjl_get_buffer, Cint, (C.PyPtr, Ptr{C.Py_buffer}, Cint)),
-        release = @cfunction(_pyjl_release_buffer, Cvoid, (C.PyPtr, Ptr{C.Py_buffer})),
-    )
-    _pyjlbase_type[] = C.PyTypeObject(
+    slots = C.PyType_Slot[
+        C.PyType_Slot(slot = C.Py_tp_dealloc,
+                      pfunc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,))),
+        C.PyType_Slot(slot = C.Py_tp_new,
+                      pfunc = @cfunction(_pyjl_new, C.PyPtr,
+                                         (C.PyPtr, C.PyPtr, C.PyPtr))),
+        C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(_pyjlbase_methods)),
+        C.PyType_Slot(slot = C.Py_bf_getbuffer,
+                      pfunc = @cfunction(_pyjl_get_buffer, Cint,
+                                         (C.PyPtr, Ptr{C.Py_buffer}, Cint))),
+        C.PyType_Slot(slot = C.Py_bf_releasebuffer,
+                      pfunc = @cfunction(_pyjl_release_buffer, Cvoid,
+                                         (C.PyPtr, Ptr{C.Py_buffer}))),
+        C.PyType_Slot(),
+    ]
+    spec = C.PyType_Spec(
         name = pointer(_pyjlbase_name),
         basicsize = sizeof(PyJuliaValueObject),
-        # new = C.POINTERS.PyType_GenericNew,
-        new = @cfunction(_pyjl_new, C.PyPtr, (C.PyPtr, C.PyPtr, C.PyPtr)),
-        dealloc = @cfunction(_pyjl_dealloc, Cvoid, (C.PyPtr,)),
+        itemsize = 0,
         flags = C.Py_TPFLAGS_BASETYPE | C.Py_TPFLAGS_HAVE_VERSION_TAG,
-        weaklistoffset = fieldoffset(PyJuliaValueObject, 3),
-        # getattro = C.POINTERS.PyObject_GenericGetAttr,
-        # setattro = C.POINTERS.PyObject_GenericSetAttr,
-        methods = pointer(_pyjlbase_methods),
-        as_buffer = pointer(_pyjlbase_as_buffer),
+        slots = pointer(slots),
     )
-    o = PyJuliaBase_Type[] = C.PyPtr(pointer(_pyjlbase_type))
-    if C.PyType_Ready(o) == -1
+    spec_ref = Ref(spec)
+    o = GC.@preserve spec slots _pyjlbase_methods spec_ref begin
+        C.PyType_FromSpec(Base.unsafe_convert(Ptr{C.PyType_Spec}, spec_ref))
+    end
+    if o == C.PyNULL
         C.PyErr_Print()
         error("Error initializing 'juliacall.ValueBase'")
     end
+    PyJuliaBase_Type[] = o
 end
 
 function __init__()

test/HeapType.jl

@@ -0,0 +1,45 @@
+@testitem "heap type from spec" begin
+    using PythonCall
+    using PythonCall.C
+
+    function _ht_answer(self::C.PyPtr, args::C.PyPtr)
+        return C.PyLong_FromLongLong(42)
+    end
+
+    meths = C.PyMethodDef[
+        C.PyMethodDef(
+            name = pointer("answer"),
+            meth = @cfunction(_ht_answer, C.PyPtr, (C.PyPtr, C.PyPtr)),
+            flags = C.Py_METH_NOARGS,
+        ),
+        C.PyMethodDef(),
+    ]
+
+    slots = C.PyType_Slot[
+        C.PyType_Slot(slot = C.Py_tp_methods, pfunc = pointer(meths)),
+        C.PyType_Slot(slot = C.Py_tp_new, pfunc = C.POINTERS.PyType_GenericNew),
+        C.PyType_Slot(),
+    ]
+
+    spec = C.PyType_Spec(
+        name = pointer("juliacall.HeapType"),
+        basicsize = sizeof(C.PyObject),
+        itemsize = 0,
+        flags = C.Py_TPFLAGS_DEFAULT | C.Py_TPFLAGS_BASETYPE,
+        slots = pointer(slots),
+    )
+
+    spec_ref = Ref(spec)
+    typ_ptr = GC.@preserve spec slots meths spec_ref begin
+        C.PyType_FromSpec(Base.unsafe_convert(Ptr{C.PyType_Spec}, spec_ref))
+    end
+    @test typ_ptr != C.PyNULL
+    typ = PythonCall.pynew(typ_ptr)
+
+    obj = pycall(typ)
+    ans = pyconvert(Int, pycall(pygetattr(obj, "answer")))
+    @test ans == 42
+
+    PythonCall.pydel!(obj)
+    PythonCall.pydel!(typ)
+end