remove reliance on PyMemoryView layout

Author: Christopher Doris

src/API/types.jl

@@ -72,14 +72,14 @@ struct PyArray{T,N,M,L,R} <: AbstractArray{T,N}
     size::NTuple{N,Int}     # size of the array
     strides::NTuple{N,Int}  # strides (in bytes) between elements
     py::Py                  # underlying python object
-    handle::Py              # the data in this array is valid as long as this handle is alive
+    handle::Any             # the data in this array is valid as long as this handle is alive
     function PyArray{T,N,M,L,R}(
         ::Val{:new},
         ptr::Ptr{R},
         size::NTuple{N,Int},
         strides::NTuple{N,Int},
         py::Py,
-        handle::Py,
+        handle::Any,
     ) where {T,N,M,L,R}
         T isa Type || error("T must be a Type")
         N isa Int || error("N must be an Int")

src/C/consts.jl

@@ -239,16 +239,6 @@ end
     internal::Ptr{Cvoid} = C_NULL
 end
 
-@kwdef struct PyMemoryViewObject
-    ob_base::PyVarObject = PyVarObject()
-    mbuf::PyPtr = PyNULL
-    hash::Py_hash_t = 0
-    flags::Cint = 0
-    exports::Py_ssize_t = 0
-    view::Py_buffer = Py_buffer()
-    weakreflist::PyPtr = PyNULL
-end
-
 @kwdef struct PyTypeObject
     ob_base::PyVarObject = PyVarObject()
     name::Cstring = C_NULL

src/C/extras.jl

@@ -7,8 +7,6 @@ Py_TypeCheckFast(o, f::Integer) = Base.GC.@preserve o PyType_IsSubtypeFast(Py_Ty
 
 PyType_IsSubtypeFast(t, f::Integer) = Cint(!iszero(PyType_GetFlags(t) & f))
 
-PyMemoryView_GET_BUFFER(m) = Base.GC.@preserve m Ptr{Py_buffer}(UnsafePtr{PyMemoryViewObject}(asptr(m)).view)
-
 PyType_CheckBuffer(t) = Base.GC.@preserve t begin
     o = Ref{PyObject}(PyObject(0, asptr(t)))
     PyObject_CheckBuffer(o)

src/JlWrap/io.jl

@@ -94,42 +94,38 @@ pyjl_handle_error_type(::typeof(pyjlbinaryio_readline), io, exc) =
     exc isa MethodError && exc.f === read ? pybuiltins.ValueError : PyNULL
 
 function pyjlbinaryio_readinto(io::IO, b::Py)
-    m = pybuiltins.memoryview(b)
-    c = m.c_contiguous
-    if !pytruth(c)
-        pydel!(c)
-        errset(pybuiltins.ValueError, "input buffer is not contiguous")
+    buf = Ref{Py_buffer}()
+    if PyObject_GetBuffer(b, buf, PyBUF_SIMPLE | PyBUF_WRITABLE) < 0
         return PyNULL
     end
-    pydel!(c)
-    buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m))
-    if buf.readonly != 0
-        pydel!(m)
-        errset(pybuiltins.ValueError, "output buffer is read-only")
-        return PyNULL
+    local nb
+    ptr = buf[].buf
+    len = buf[].len
+    try
+        data = unsafe_wrap(Array, Ptr{UInt8}(ptr), len)
+        nb = readbytes!(io, data)
+    finally
+        PyBuffer_Release(buf)
     end
-    data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len)
-    nb = readbytes!(io, data)
-    pydel!(m)
     return Py(nb)
 end
 pyjl_handle_error_type(::typeof(pyjlbinaryio_readinto), io, exc) =
     exc isa MethodError && exc.f === readbytes! ? pybuiltins.ValueError : PyNULL
 
 function pyjlbinaryio_write(io::IO, b::Py)
-    m = pybuiltins.memoryview(b)
-    c = m.c_contiguous
-    if !pytruth(c)
-        pydel!(c)
-        errset(pybuiltins.ValueError, "input buffer is not contiguous")
+    buf = Ref{Py_buffer}()
+    if PyObject_GetBuffer(b, buf, PyBUF_SIMPLE) < 0
         return PyNULL
     end
-    pydel!(c)
-    buf = unsafe_load(C.PyMemoryView_GET_BUFFER(m))
-    data = unsafe_wrap(Array, Ptr{UInt8}(buf.buf), buf.len)
-    write(io, data)
-    pydel!(m)
-    return Py(buf.len)
+    ptr = buf[].buf
+    len = buf[].len
+    try
+        data = unsafe_wrap(Array, Ptr{UInt8}(ptr), len)
+        write(io, data)        
+    finally
+        PyBuffer_Release(buf)
+    end
+    return Py(len)
 end
 pyjl_handle_error_type(::typeof(pyjlbinaryio_write), io, exc) =
     exc isa MethodError && exc.f === write ? pybuiltins.ValueError : PyNULL

src/Wrap/PyArray.jl

@@ -187,7 +187,7 @@ struct PyArraySource_ArrayInterface <: PyArraySource
     dict::Py
     ptr::Ptr{Cvoid}
     readonly::Bool
-    handle::Py
+    handle::Any
 end
 function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__)
     # offset
@@ -202,14 +202,16 @@ function PyArraySource_ArrayInterface(x::Py, d::Py = x.__array_interface__)
         ptr = Ptr{Cvoid}(pyconvert(UInt, data[0]))
         readonly = pyconvert(Bool, data[1])
         pydel!(data)
-        handle = Py((x, d))
+        handle = (x, d)
     else
-        memview = @py memoryview(data === None ? x : data)
-        pydel!(data)
-        buf = UnsafePtr(C.PyMemoryView_GET_BUFFER(memview))
-        ptr = buf.buf[!]
-        readonly = buf.readonly[] != 0
-        handle = Py((x, memview))
+        buf = Ref{C.Py_buffer}()
+        if C.PyObject_GetBuffer(data === None ? x : data, buf, C.PyBUF_RECORDS) < 0
+            pythrow()
+        end
+        finalizer(C.PyBuffer_Release, buf)
+        ptr = buf[].buf
+        readonly = buf[].readonly != 0
+        handle = buf
     end
     PyArraySource_ArrayInterface(x, d, ptr, readonly, handle)
 end
@@ -525,13 +527,15 @@ end
 
 struct PyArraySource_Buffer <: PyArraySource
     obj::Py
-    memview::Py
-    buf::C.UnsafePtr{C.Py_buffer}
+    buf::Base.RefValue{C.Py_buffer}
 end
 function PyArraySource_Buffer(x::Py)
-    memview = pybuiltins.memoryview(x)
-    buf = C.UnsafePtr(C.PyMemoryView_GET_BUFFER(memview))
-    PyArraySource_Buffer(x, memview, buf)
+    buf = Ref{C.Py_buffer}()
+    if C.PyObject_GetBuffer(x, buf, C.PyBUF_RECORDS) < 0
+        pythrow()
+    end
+    finalizer(C.PyBuffer_Release, buf)
+    PyArraySource_Buffer(x, buf)
 end
 
 const PYARRAY_BUFFERFORMAT_TO_TYPE = let c = Utils.islittleendian() ? '<' : '>'
@@ -578,20 +582,20 @@ pyarray_bufferformat_to_type(fmt::String) = get(
 )
 
 function pyarray_get_R(src::PyArraySource_Buffer)
-    ptr = src.buf.format[]
-    return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(String(ptr))
+    ptr = src.buf[].format
+    return ptr == C_NULL ? UInt8 : pyarray_bufferformat_to_type(unsafe_string(ptr))
 end
 
-pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf.buf[!])
+pyarray_get_ptr(src::PyArraySource_Buffer, ::Type{R}) where {R} = Ptr{R}(src.buf[].buf)
 
-pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf.ndim[])
+pyarray_get_N(src::PyArraySource_Buffer) = Int(src.buf[].ndim)
 
 function pyarray_get_size(src::PyArraySource_Buffer, ::Val{N}) where {N}
-    size = src.buf.shape[]
+    size = src.buf[].shape
     if size == C_NULL
-        N == 0 ? () : N == 1 ? (Int(src.buf.len[]),) : @assert false
+        N == 0 ? () : N == 1 ? (Int(src.buf[].len),) : @assert false
     else
-        ntuple(i -> Int(size[i]), N)
+        ntuple(i -> Int(unsafe_load(size, i)), N)
     end
 end
 
@@ -601,18 +605,18 @@ function pyarray_get_strides(
     ::Type{R},
     size::NTuple{N,Int},
 ) where {N,R}
-    strides = src.buf.strides[]
+    strides = src.buf[].strides
     if strides == C_NULL
-        itemsize = src.buf.shape[] == C_NULL ? 1 : src.buf.itemsize[]
+        itemsize = src.buf[].shape == C_NULL ? 1 : src.buf[].itemsize
         Utils.size_to_cstrides(itemsize, size)
     else
-        ntuple(i -> Int(strides[i]), N)
+        ntuple(i -> Int(unsafe_load(strides, i)), N)
     end
 end
 
-pyarray_get_M(src::PyArraySource_Buffer) = src.buf.readonly[] == 0
+pyarray_get_M(src::PyArraySource_Buffer) = src.buf[].readonly == 0
 
-pyarray_get_handle(src::PyArraySource_Buffer) = src.memview
+pyarray_get_handle(src::PyArraySource_Buffer) = src.buf
 
 # AbstractArray methods