Skip to content

Commit 1640549

Browse files
msimacekmsimacek
committed
[GR-23189][GR-23247] Intrinsify memoryview and make more tests pass
PullRequest: graalpython/1348
2 parents 6a402ae + 04c8653 commit 1640549

36 files changed

+3777
-3611
lines changed

‎graalpython/com.oracle.graal.python.cext/modules/_memoryview.c

Lines changed: 0 additions & 3296 deletions
This file was deleted.

‎graalpython/com.oracle.graal.python.cext/setup.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,6 @@ def __call__(self):
349349
builtin_exts = (
350350
NativeBuiltinModule("_cpython_sre"),
351351
NativeBuiltinModule("_cpython_unicodedata"),
352-
NativeBuiltinModule("_memoryview"),
353352
NativeBuiltinModule("_mmap"),
354353
NativeBuiltinModule("_cpython_struct"),
355354
NativeBuiltinModule("_testcapi"),

‎graalpython/com.oracle.graal.python.cext/src/capi.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ initialize_type(_PyWeakref_CallableProxyType, CallableProxyType, PyWeakReference
211211

212212
POLYGLOT_DECLARE_TYPE(PyThreadState);
213213
POLYGLOT_DECLARE_TYPE(newfunc);
214+
POLYGLOT_DECLARE_TYPE(Py_buffer);
214215

215216

216217
static void initialize_globals() {
@@ -248,6 +249,7 @@ static void initialize_bufferprocs() {
248249
polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyBytes_Type), (getbufferproc)bytes_buffer_getbuffer, (releasebufferproc)NULL);
249250
polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyByteArray_Type), (getbufferproc)bytearray_getbuffer, (releasebufferproc)NULL);
250251
polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyBuffer_Type), (getbufferproc)bufferdecorator_getbuffer, (releasebufferproc)NULL);
252+
polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_SetBufferProcs", native_to_java((PyObject*)&PyMemoryView_Type), (getbufferproc)memoryview_getbuffer, (releasebufferproc)memoryview_releasebuffer);
251253
}
252254

253255
__attribute__((constructor (20000)))
@@ -741,6 +743,10 @@ int truffle_ptr_compare(void* x, void* y, int op) {
741743
}
742744
}
743745

746+
void* truffle_ptr_add(void* x, Py_ssize_t y) {
747+
return x + y;
748+
}
749+
744750
double truffle_read_ob_fval(PyFloatObject* fobj) {
745751
return fobj->ob_fval;
746752
}

‎graalpython/com.oracle.graal.python.cext/src/capi.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -394,6 +394,8 @@ int bytes_copy2mem(char* target, char* source, size_t nbytes);
394394

395395
/* MEMORYVIEW, BUFFERDECORATOR */
396396
int bufferdecorator_getbuffer(PyBufferDecorator *self, Py_buffer *view, int flags);
397+
int memoryview_getbuffer(PyMemoryViewObject *self, Py_buffer *view, int flags);
398+
void memoryview_releasebuffer(PyMemoryViewObject *self, Py_buffer *view);
397399

398400
typedef PyObject* PyObjectPtr;
399401
POLYGLOT_DECLARE_TYPE(PyObjectPtr);

‎graalpython/com.oracle.graal.python.cext/src/memoryobject.c

Lines changed: 225 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -40,16 +40,236 @@
4040
*/
4141
#include "capi.h"
4242

43+
#if SIZEOF_SIZE_T == 8
44+
#define polyglot_from_size_array polyglot_from_i64_array
45+
#elif SIZEOF_SIZE_T == 4
46+
#define polyglot_from_size_array polyglot_from_i32_array
47+
#endif
48+
49+
/* Macros taken from CPython */
50+
/* Memoryview buffer properties */
51+
#define MV_C_CONTIGUOUS(flags) (flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_C))
52+
#define MV_F_CONTIGUOUS(flags) \
53+
(flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_FORTRAN))
54+
#define MV_ANY_CONTIGUOUS(flags) \
55+
(flags&(_Py_MEMORYVIEW_SCALAR|_Py_MEMORYVIEW_C|_Py_MEMORYVIEW_FORTRAN))
56+
57+
/* getbuffer() requests */
58+
#define REQ_INDIRECT(flags) ((flags&PyBUF_INDIRECT) == PyBUF_INDIRECT)
59+
#define REQ_C_CONTIGUOUS(flags) ((flags&PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS)
60+
#define REQ_F_CONTIGUOUS(flags) ((flags&PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS)
61+
#define REQ_ANY_CONTIGUOUS(flags) ((flags&PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS)
62+
#define REQ_STRIDES(flags) ((flags&PyBUF_STRIDES) == PyBUF_STRIDES)
63+
#define REQ_SHAPE(flags) ((flags&PyBUF_ND) == PyBUF_ND)
64+
#define REQ_WRITABLE(flags) (flags&PyBUF_WRITABLE)
65+
#define REQ_FORMAT(flags) (flags&PyBUF_FORMAT)
66+
67+
#define BASE_INACCESSIBLE(mv) \
68+
(((PyMemoryViewObject *)mv)->flags&_Py_MEMORYVIEW_RELEASED)
69+
#define CHECK_RELEASED(mv) \
70+
if (BASE_INACCESSIBLE(mv)) { \
71+
PyErr_SetString(PyExc_ValueError, \
72+
"operation forbidden on released memoryview object"); \
73+
return NULL; \
74+
}
75+
#define CHECK_RELEASED_INT(mv) \
76+
if (BASE_INACCESSIBLE(mv)) { \
77+
PyErr_SetString(PyExc_ValueError, \
78+
"operation forbidden on released memoryview object"); \
79+
return -1; \
80+
}
81+
4382
PyTypeObject PyMemoryView_Type = PY_TRUFFLE_TYPE_WITH_ITEMSIZE("memoryview", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, offsetof(PyMemoryViewObject, ob_array), sizeof(Py_ssize_t));
4483
PyTypeObject PyBuffer_Type = PY_TRUFFLE_TYPE("buffer", &PyType_Type, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, sizeof(PyBufferDecorator));
4584

4685
int bufferdecorator_getbuffer(PyBufferDecorator *self, Py_buffer *view, int flags) {
4786
return PyBuffer_FillInfo(view, (PyObject*)self, polyglot_get_member(self, "buf_delegate"), PyObject_Size((PyObject *)self) * sizeof(PyObject*), self->readonly, flags);
4887
}
4988

50-
PyObject * PyMemoryView_FromObject(PyObject *v) {
51-
// TODO(fa): This needs to be fixed. The actual implementation is located in
52-
// '_memoryview.c'. However, the current way we use it does not allow C exts
53-
// to link to it. We need to restructure this.
54-
return NULL;
89+
/* called from memoryview implementation to do pointer arithmetics currently not possible from Java */
90+
int8_t* truffle_add_suboffset(int8_t *ptr, Py_ssize_t offset, Py_ssize_t suboffset, Py_ssize_t remaining_length) {
91+
return polyglot_from_i8_array(*(int8_t**)(ptr + offset) + suboffset, remaining_length);
92+
}
93+
94+
UPCALL_ID(PyMemoryView_FromObject)
95+
PyObject* PyMemoryView_FromObject(PyObject *v) {
96+
return UPCALL_CEXT_O(_jls_PyMemoryView_FromObject, native_to_java(v));
97+
}
98+
99+
/* called back from the above upcall only if the object was native */
100+
PyObject* PyTruffle_MemoryViewFromObject(PyObject *v) {
101+
if (PyObject_CheckBuffer(v)) {
102+
Py_buffer* buffer = malloc(sizeof(Py_buffer));
103+
if (PyObject_GetBuffer(v, buffer, PyBUF_FULL_RO) < 0) {
104+
return NULL;
105+
}
106+
Py_ssize_t ndim = buffer->ndim;
107+
int needs_release = 0;
108+
if (buffer->obj != NULL) {
109+
PyBufferProcs *pb;
110+
pb = Py_TYPE(buffer->obj)->tp_as_buffer;
111+
if (pb) {
112+
needs_release = pb->bf_releasebuffer != NULL;
113+
}
114+
}
115+
PyObject *mv = polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_MemoryViewFromBuffer",
116+
needs_release ? buffer : NULL, /* We only need the ptr for the release */
117+
native_to_java(buffer->obj),
118+
buffer->len,
119+
buffer->readonly,
120+
buffer->itemsize,
121+
polyglot_from_string(buffer->format ? buffer->format : "B", "ascii"),
122+
buffer->ndim,
123+
polyglot_from_i8_array(buffer->buf, buffer->len),
124+
buffer->shape ? polyglot_from_size_array(buffer->shape, ndim) : NULL,
125+
buffer->strides ? polyglot_from_size_array(buffer->strides, ndim) : NULL,
126+
buffer->suboffsets ? polyglot_from_size_array(buffer->suboffsets, ndim) : NULL);
127+
if (!needs_release) {
128+
free(buffer);
129+
}
130+
return mv;
131+
}
132+
133+
PyErr_Format(PyExc_TypeError,
134+
"memoryview: a bytes-like object is required, not '%.200s'",
135+
Py_TYPE(v)->tp_name);
136+
return NULL;
137+
}
138+
139+
/* Release buffer struct allocated in PyTruffle_MemoryViewFromObject */
140+
void PyTruffle_ReleaseBuffer(Py_buffer* buffer) {
141+
if (buffer->obj != NULL) {
142+
PyBufferProcs *pb;
143+
pb = Py_TYPE(buffer->obj)->tp_as_buffer;
144+
if (pb) {
145+
pb->bf_releasebuffer(buffer->obj, buffer);
146+
}
147+
}
148+
free(buffer);
149+
}
150+
151+
PyObject* PyMemoryView_FromBuffer(Py_buffer *buffer) {
152+
Py_ssize_t ndim = buffer->ndim;
153+
if (buffer->buf == NULL) {
154+
PyErr_SetString(PyExc_ValueError,
155+
"PyMemoryView_FromBuffer(): info->buf must not be NULL");
156+
return NULL;
157+
}
158+
return polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_MemoryViewFromBuffer",
159+
NULL,
160+
NULL,
161+
buffer->len,
162+
buffer->readonly,
163+
buffer->itemsize,
164+
polyglot_from_string(buffer->format ? buffer->format : "B", "ascii"),
165+
buffer->ndim,
166+
polyglot_from_i8_array(buffer->buf, buffer->len),
167+
buffer->shape ? polyglot_from_size_array(buffer->shape, ndim) : NULL,
168+
buffer->strides ? polyglot_from_size_array(buffer->strides, ndim) : NULL,
169+
buffer->suboffsets ? polyglot_from_size_array(buffer->suboffsets, ndim) : NULL);
170+
}
171+
172+
PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) {
173+
assert(mem != NULL);
174+
assert(flags == PyBUF_READ || flags == PyBUF_WRITE);
175+
int readonly = (flags == PyBUF_WRITE) ? 0 : 1;
176+
return polyglot_invoke(PY_TRUFFLE_CEXT, "PyTruffle_MemoryViewFromBuffer",
177+
NULL, NULL, size, readonly, 1, polyglot_from_string("B", "ascii"), 1, polyglot_from_i8_array((int8_t*)mem, size), NULL, NULL, NULL);
178+
}
179+
180+
UPCALL_ID(PyMemoryView_GetContiguous)
181+
PyObject* PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) {
182+
return UPCALL_CEXT_O(_jls_PyMemoryView_GetContiguous, native_to_java(obj), buffertype, (int)order);
183+
}
184+
185+
/* Taken from CPython memoryobject.c: memory_getbuf */
186+
int memoryview_getbuffer(PyMemoryViewObject *self, Py_buffer *view, int flags)
187+
{
188+
Py_buffer *base = &self->view;
189+
int baseflags = self->flags;
190+
191+
CHECK_RELEASED_INT(self);
192+
193+
/* start with complete information */
194+
//*view = *base;
195+
view->buf = base->buf;
196+
view->format = base->format;
197+
view->itemsize = base->itemsize;
198+
view->len = base->len;
199+
view->ndim = base->ndim;
200+
view->readonly = base->readonly;
201+
view->shape = base->shape;
202+
view->strides = base->strides;
203+
view->suboffsets = base->suboffsets;
204+
view->obj = NULL;
205+
206+
if (REQ_WRITABLE(flags) && base->readonly) {
207+
PyErr_SetString(PyExc_BufferError,
208+
"memoryview: underlying buffer is not writable");
209+
return -1;
210+
}
211+
if (!REQ_FORMAT(flags)) {
212+
/* NULL indicates that the buffer's data type has been cast to 'B'.
213+
view->itemsize is the _previous_ itemsize. If shape is present,
214+
the equality product(shape) * itemsize = len still holds at this
215+
point. The equality calcsize(format) = itemsize does _not_ hold
216+
from here on! */
217+
view->format = NULL;
218+
}
219+
220+
if (REQ_C_CONTIGUOUS(flags) && !MV_C_CONTIGUOUS(baseflags)) {
221+
PyErr_SetString(PyExc_BufferError,
222+
"memoryview: underlying buffer is not C-contiguous");
223+
return -1;
224+
}
225+
if (REQ_F_CONTIGUOUS(flags) && !MV_F_CONTIGUOUS(baseflags)) {
226+
PyErr_SetString(PyExc_BufferError,
227+
"memoryview: underlying buffer is not Fortran contiguous");
228+
return -1;
229+
}
230+
if (REQ_ANY_CONTIGUOUS(flags) && !MV_ANY_CONTIGUOUS(baseflags)) {
231+
PyErr_SetString(PyExc_BufferError,
232+
"memoryview: underlying buffer is not contiguous");
233+
return -1;
234+
}
235+
if (!REQ_INDIRECT(flags) && (baseflags & _Py_MEMORYVIEW_PIL)) {
236+
PyErr_SetString(PyExc_BufferError,
237+
"memoryview: underlying buffer requires suboffsets");
238+
return -1;
239+
}
240+
if (!REQ_STRIDES(flags)) {
241+
if (!MV_C_CONTIGUOUS(baseflags)) {
242+
PyErr_SetString(PyExc_BufferError,
243+
"memoryview: underlying buffer is not C-contiguous");
244+
return -1;
245+
}
246+
view->strides = NULL;
247+
}
248+
if (!REQ_SHAPE(flags)) {
249+
/* PyBUF_SIMPLE or PyBUF_WRITABLE: at this point buf is C-contiguous,
250+
so base->buf = ndbuf->data. */
251+
if (view->format != NULL) {
252+
/* PyBUF_SIMPLE|PyBUF_FORMAT and PyBUF_WRITABLE|PyBUF_FORMAT do
253+
not make sense. */
254+
PyErr_Format(PyExc_BufferError,
255+
"memoryview: cannot cast to unsigned bytes if the format flag "
256+
"is present");
257+
return -1;
258+
}
259+
/* product(shape) * itemsize = len and calcsize(format) = itemsize
260+
do _not_ hold from here on! */
261+
view->ndim = 1;
262+
view->shape = NULL;
263+
}
264+
265+
266+
view->obj = (PyObject *)self;
267+
Py_INCREF(view->obj);
268+
self->exports++;
269+
270+
return 0;
271+
}
272+
273+
void memoryview_releasebuffer(PyMemoryViewObject *self, Py_buffer *view) {
274+
self->exports--;
55275
}

‎graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,11 @@ def unhandled_error_compare(x, y):
5454
else:
5555
return x == y
5656

57+
def unhandled_error_compare_with_message(x, y):
58+
if (isinstance(x, BaseException) and isinstance(y, BaseException)):
59+
return type(x) == type(y) and str(x) == str(y)
60+
else:
61+
return x == y
5762

5863
class CPyExtTestCase():
5964

0 commit comments

Comments
 (0)