[GR-34614][GR-44418] More fixes for PyTorch

PullRequest: graalpython/2825

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/cpython/code.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022, Oracle and/or its affiliates.
+/* Copyright (c) 2022, 2023, Oracle and/or its affiliates.
  * Copyright (C) 1996-2022 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -139,6 +139,10 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
    use PyFrame_GetLineNumber() instead. */
 PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
 
+/* GraalPy specific to avoid direct field access. Used by (at least) pytorch */
+PyAPI_FUNC(PyObject*) PyCode_GetFileName(PyCodeObject* code);
+PyAPI_FUNC(PyObject*) PyCode_GetName(PyCodeObject* code);
+
 /* for internal use only */
 struct _opaque {
     int computed_line;

graalpython/com.oracle.graal.python.cext/include/cpython/pyframe.h

@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * The Universal Permissive License (UPL), Version 1.0
+ *
+ * Subject to the condition set forth below, permission is hereby granted to any
+ * person obtaining a copy of this software, associated documentation and/or
+ * data (collectively the "Software"), free of charge and under any and all
+ * copyright rights in the Software, and any and all patent rights owned or
+ * freely licensable by each licensor hereunder covering either (i) the
+ * unmodified Software as contributed to or provided by such licensor, or (ii)
+ * the Larger Works (as defined below), to deal in both
+ *
+ * (a) the Software, and
+ *
+ * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+ * one is included with the Software each a "Larger Work" to which the Software
+ * is contributed by such licensors),
+ *
+ * without restriction, including without limitation the rights to copy, create
+ * derivative works of, display, perform, and distribute the Software and make,
+ * use, sell, offer for sale, import, export, have made, and have sold the
+ * Software and the Larger Work(s), and to sublicense the foregoing rights on
+ * either these or other terms.
+ *
+ * This license is subject to the following condition:
+ *
+ * The above copyright notice and either this complete permission notice or at a
+ * minimum a reference to the UPL must be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef Py_CPYTHON_PYFRAME_H
+#  error "this header file must not be included directly"
+#endif
+
+PyAPI_FUNC(PyFrameObject *) PyFrame_GetBack(PyFrameObject *frame);
+PyAPI_FUNC(PyObject *) PyFrame_GetLocals(PyFrameObject *frame);
+
+PyAPI_FUNC(PyObject *) PyFrame_GetGlobals(PyFrameObject *frame);
+PyAPI_FUNC(PyObject *) PyFrame_GetBuiltins(PyFrameObject *frame);
+
+PyAPI_FUNC(PyObject *) PyFrame_GetGenerator(PyFrameObject *frame);
+PyAPI_FUNC(int) PyFrame_GetLasti(PyFrameObject *frame);

graalpython/com.oracle.graal.python.cext/include/pyframe.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022, Oracle and/or its affiliates.
+/* Copyright (c) 2022, 2023, Oracle and/or its affiliates.
  * Copyright (C) 1996-2022 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -21,6 +21,12 @@ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *);
 
 PyAPI_FUNC(PyCodeObject *) PyFrame_GetCode(PyFrameObject *frame);
 
+#ifndef Py_LIMITED_API
+#  define Py_CPYTHON_PYFRAME_H
+#  include "cpython/pyframe.h"
+#  undef Py_CPYTHON_PYFRAME_H
+#endif
+
 #ifdef __cplusplus
 }
 #endif

graalpython/com.oracle.graal.python.cext/src/_warnings.c

@@ -54,6 +54,12 @@ MUST_INLINE static int warn_unicode(PyObject *category, PyObject *message, Py_ss
     return 0;
 }
 
+MUST_INLINE static PyObject* warn_explicit(PyObject *category, PyObject *message,
+              PyObject *filename, int lineno,
+              PyObject *module, PyObject *registry) {
+    return GraalPyTruffleErr_WarnExplicit(category, message, filename, lineno, module, registry);
+}
+
 // taken from CPython "Python/_warnings.c"
 int PyErr_WarnEx(PyObject *category, const char *text, Py_ssize_t stack_level) {
     PyObject *message = PyUnicode_FromString(text);
@@ -84,3 +90,90 @@ int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *
     return ret;
 }
 
+// Adapted from CPython
+int PyErr_WarnExplicitObject(PyObject *category, PyObject *message,
+                         PyObject *filename, int lineno,
+                         PyObject *module, PyObject *registry)
+{
+    PyObject *res;
+    if (category == NULL)
+        category = PyExc_RuntimeWarning;
+    res = warn_explicit(category, message, filename, lineno,
+                        module, registry);
+    if (res == NULL)
+        return -1;
+    Py_DECREF(res);
+    return 0;
+}
+
+// Adapted from CPython
+int PyErr_WarnExplicit(PyObject *category, const char *text,
+                   const char *filename_str, int lineno,
+                   const char *module_str, PyObject *registry)
+{
+    PyObject *message = PyUnicode_FromString(text);
+    PyObject *filename = PyUnicode_DecodeFSDefault(filename_str);
+    PyObject *module = NULL;
+    int ret = -1;
+
+    if (message == NULL || filename == NULL)
+        goto exit;
+    if (module_str != NULL) {
+        module = PyUnicode_FromString(module_str);
+        if (module == NULL)
+            goto exit;
+    }
+
+    ret = PyErr_WarnExplicitObject(category, message, filename, lineno,
+                                   module, registry);
+
+ exit:
+    Py_XDECREF(message);
+    Py_XDECREF(module);
+    Py_XDECREF(filename);
+    return ret;
+}
+
+// Adapted from CPython
+int PyErr_WarnExplicitFormat(PyObject *category,
+                         const char *filename_str, int lineno,
+                         const char *module_str, PyObject *registry,
+                         const char *format, ...)
+{
+    PyObject *message;
+    PyObject *module = NULL;
+    PyObject *filename = PyUnicode_DecodeFSDefault(filename_str);
+    int ret = -1;
+    va_list vargs;
+
+    if (filename == NULL)
+        goto exit;
+    if (module_str != NULL) {
+        module = PyUnicode_FromString(module_str);
+        if (module == NULL)
+            goto exit;
+    }
+
+#ifdef HAVE_STDARG_PROTOTYPES
+    va_start(vargs, format);
+#else
+    va_start(vargs);
+#endif
+    message = PyUnicode_FromFormatV(format, vargs);
+    if (message != NULL) {
+        PyObject *res = warn_explicit(category, message, filename, lineno,
+                            module, registry);
+        Py_DECREF(message);
+        if (res != NULL) {
+            Py_DECREF(res);
+            ret = 0;
+        }
+    }
+    va_end(vargs);
+exit:
+    Py_XDECREF(module);
+    Py_XDECREF(filename);
+    return ret;
+}
+
+

graalpython/com.oracle.graal.python.cext/src/capi.c

@@ -1505,6 +1505,18 @@ PyAPI_FUNC(int) PyCapsule_SetPointer(PyObject* a, void* b) {
 PyAPI_FUNC(PyObject*) PyClassMethod_New(PyObject* a) {
     return GraalPyClassMethod_New(a);
 }
+#undef PyCode_Addr2Line
+PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject* a, int b) {
+    return GraalPyCode_Addr2Line(a, b);
+}
+#undef PyCode_GetFileName
+PyAPI_FUNC(PyObject*) PyCode_GetFileName(PyCodeObject* a) {
+    return GraalPyCode_GetFileName(a);
+}
+#undef PyCode_GetName
+PyAPI_FUNC(PyObject*) PyCode_GetName(PyCodeObject* a) {
+    return GraalPyCode_GetName(a);
+}
 #undef PyCode_New
 PyAPI_FUNC(PyCodeObject*) PyCode_New(int a, int b, int c, int d, int e, PyObject* f, PyObject* g, PyObject* h, PyObject* i, PyObject* j, PyObject* k, PyObject* l, PyObject* m, int n, PyObject* o) {
     return GraalPyCode_New(a, b, c, d, e, f, g, h, i, j, k, l, m, n, o);
@@ -1633,14 +1645,14 @@ PyAPI_FUNC(void) PyErr_Restore(PyObject* a, PyObject* b, PyObject* c) {
 PyAPI_FUNC(void) PyErr_SetExcInfo(PyObject* a, PyObject* b, PyObject* c) {
     GraalPyErr_SetExcInfo(a, b, c);
 }
-#undef PyErr_WarnExplicit
-PyAPI_FUNC(int) PyErr_WarnExplicit(PyObject* a, const char* b, const char* c, int d, const char* e, PyObject* f) {
-    return GraalPyErr_WarnExplicit(a, truffleString(b), truffleString(c), d, truffleString(e), f);
-}
 #undef PyEval_GetBuiltins
 PyAPI_FUNC(PyObject*) PyEval_GetBuiltins() {
     return GraalPyEval_GetBuiltins();
 }
+#undef PyEval_GetFrame
+PyAPI_FUNC(PyFrameObject*) PyEval_GetFrame() {
+    return GraalPyEval_GetFrame();
+}
 #undef PyEval_RestoreThread
 PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState* a) {
     GraalPyEval_RestoreThread(a);
@@ -1677,6 +1689,34 @@ PyAPI_FUNC(int) PyFile_WriteObject(PyObject* a, PyObject* b, int c) {
 PyAPI_FUNC(PyObject*) PyFloat_FromDouble(double a) {
     return GraalPyFloat_FromDouble(a);
 }
+#undef PyFrame_GetBack
+PyAPI_FUNC(PyFrameObject*) PyFrame_GetBack(PyFrameObject* a) {
+    return GraalPyFrame_GetBack(a);
+}
+#undef PyFrame_GetBuiltins
+PyAPI_FUNC(PyObject*) PyFrame_GetBuiltins(PyFrameObject* a) {
+    return GraalPyFrame_GetBuiltins(a);
+}
+#undef PyFrame_GetCode
+PyAPI_FUNC(PyCodeObject*) PyFrame_GetCode(PyFrameObject* a) {
+    return GraalPyFrame_GetCode(a);
+}
+#undef PyFrame_GetGlobals
+PyAPI_FUNC(PyObject*) PyFrame_GetGlobals(PyFrameObject* a) {
+    return GraalPyFrame_GetGlobals(a);
+}
+#undef PyFrame_GetLasti
+PyAPI_FUNC(int) PyFrame_GetLasti(PyFrameObject* a) {
+    return GraalPyFrame_GetLasti(a);
+}
+#undef PyFrame_GetLineNumber
+PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject* a) {
+    return GraalPyFrame_GetLineNumber(a);
+}
+#undef PyFrame_GetLocals
+PyAPI_FUNC(PyObject*) PyFrame_GetLocals(PyFrameObject* a) {
+    return GraalPyFrame_GetLocals(a);
+}
 #undef PyFrame_New
 PyAPI_FUNC(PyFrameObject*) PyFrame_New(PyThreadState* a, PyCodeObject* b, PyObject* c, PyObject* d) {
     return GraalPyFrame_New(a, b, c, d);

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -114,6 +114,9 @@ typedef struct {
     BUILTIN(PyCapsule_SetName, int, PyObject*, const char*) \
     BUILTIN(PyCapsule_SetPointer, int, PyObject*, void*) \
     BUILTIN(PyClassMethod_New, PyObject*, PyObject*) \
+    BUILTIN(PyCode_Addr2Line, int, PyCodeObject*, int) \
+    BUILTIN(PyCode_GetFileName, PyObject*, PyCodeObject*) \
+    BUILTIN(PyCode_GetName, PyObject*, PyCodeObject*) \
     BUILTIN(PyCode_New, PyCodeObject*, int, int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, int, PyObject*) \
     BUILTIN(PyCode_NewEmpty, PyCodeObject*, const char*, const char*, int) \
     BUILTIN(PyCode_NewWithPosOnlyArgs, PyCodeObject*, int, int, int, int, int, int, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, PyObject*, int, PyObject*) \
@@ -146,8 +149,8 @@ typedef struct {
     BUILTIN(PyErr_PrintEx, void, int) \
     BUILTIN(PyErr_Restore, void, PyObject*, PyObject*, PyObject*) \
     BUILTIN(PyErr_SetExcInfo, void, PyObject*, PyObject*, PyObject*) \
-    BUILTIN(PyErr_WarnExplicit, int, PyObject*, const char*, const char*, int, const char*, PyObject*) \
     BUILTIN(PyEval_GetBuiltins, PyObject*) \
+    BUILTIN(PyEval_GetFrame, PyFrameObject*) \
     BUILTIN(PyEval_RestoreThread, void, PyThreadState*) \
     BUILTIN(PyEval_SaveThread, PyThreadState*) \
     BUILTIN(PyException_GetCause, PyObject*, PyObject*) \
@@ -157,6 +160,13 @@ typedef struct {
     BUILTIN(PyException_SetTraceback, int, PyObject*, PyObject*) \
     BUILTIN(PyFile_WriteObject, int, PyObject*, PyObject*, int) \
     BUILTIN(PyFloat_FromDouble, PyObject*, double) \
+    BUILTIN(PyFrame_GetBack, PyFrameObject*, PyFrameObject*) \
+    BUILTIN(PyFrame_GetBuiltins, PyObject*, PyFrameObject*) \
+    BUILTIN(PyFrame_GetCode, PyCodeObject*, PyFrameObject*) \
+    BUILTIN(PyFrame_GetGlobals, PyObject*, PyFrameObject*) \
+    BUILTIN(PyFrame_GetLasti, int, PyFrameObject*) \
+    BUILTIN(PyFrame_GetLineNumber, int, PyFrameObject*) \
+    BUILTIN(PyFrame_GetLocals, PyObject*, PyFrameObject*) \
     BUILTIN(PyFrame_New, PyFrameObject*, PyThreadState*, PyCodeObject*, PyObject*, PyObject*) \
     BUILTIN(PyFrozenSet_New, PyObject*, PyObject*) \
     BUILTIN(PyGILState_Check, int) \
@@ -302,6 +312,7 @@ typedef struct {
     BUILTIN(PyTruffleDict_Next, PyObject*, PyObject*, Py_ssize_t) \
     BUILTIN(PyTruffleErr_Fetch, PyObject*) \
     BUILTIN(PyTruffleErr_GetExcInfo, PyObject*) \
+    BUILTIN(PyTruffleErr_WarnExplicit, PyObject*, PyObject*, PyObject*, PyObject*, int, PyObject*, PyObject*) \
     BUILTIN(PyTruffleFloat_AsDouble, double, PyObject*) \
     BUILTIN(PyTruffleHash_InitSecret, void, void*) \
     BUILTIN(PyTruffleLong_AsPrimitive, long, PyObject*, int, long) \
@@ -353,7 +364,7 @@ typedef struct {
     BUILTIN(PyTruffle_NoValue, PyObject*) \
     BUILTIN(PyTruffle_None, PyObject*) \
     BUILTIN(PyTruffle_NotImplemented, PyObject*) \
-    BUILTIN(PyTruffle_Object_Free, int, void*) \
+    BUILTIN(PyTruffle_Object_Free, void, void*) \
     BUILTIN(PyTruffle_Register_NULL, void, void*) \
     BUILTIN(PyTruffle_Set_Native_Slots, int, PyTypeObject*, void*, void*) \
     BUILTIN(PyTruffle_Set_SulongType, void*, PyTypeObject*, void*) \

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -79,7 +79,6 @@ int _PyDict_Next(PyObject *d, Py_ssize_t *ppos, PyObject **pkey, PyObject **pval
     	}
     	return 0;
     }
-    (*ppos)++;
     if (pkey != NULL) {
     	*pkey = PyTuple_GetItem(tresult, 0);
     }
@@ -89,6 +88,7 @@ int _PyDict_Next(PyObject *d, Py_ssize_t *ppos, PyObject **pkey, PyObject **pval
     if (phash != NULL) {
     	*phash = PyLong_AsSsize_t(PyTuple_GetItem(tresult, 2));
     }
+    *ppos = PyLong_AsSsize_t(PyTuple_GetItem(tresult, 3));
     Py_DECREF(tresult);
     return 1;
 

graalpython/com.oracle.graal.python.cext/src/object_shared.c

@@ -215,6 +215,10 @@ PyObject* PyObject_CallObject(PyObject* callable, PyObject* args) {
 	return Graal_PyTruffleObject_Call1(callable, args, NULL, 0);
 }
 
+PyObject* PyObject_CallNoArgs(PyObject* callable) {
+    return Graal_PyTruffleObject_Call1(callable, NULL, NULL, 0);
+}
+
 PyObject* PyObject_CallFunction(PyObject* callable, const char* fmt, ...) {
     if (fmt == NULL || fmt[0] == '\0') {
 	    return Graal_PyTruffleObject_Call1(callable, NULL, NULL, 0);

graalpython/com.oracle.graal.python.cext/src/obmalloc.c

@@ -119,14 +119,12 @@ void _PyObject_Free(void* ptr) {
 		return;
 	}
 	if(points_to_py_handle_space(ptr) || polyglot_is_value(ptr)) {
-		if(GraalPyTruffle_Object_Free(ptr)) {
-		    /* If 1 is returned, the upcall function already took care of freeing */
-		    return;
-		}
+	    GraalPyTruffle_Object_Free(ptr);
+	} else {
+        mem_head_t* ptr_with_head = AS_MEM_HEAD(ptr);
+        PyTruffle_FreeMemory(ptr_with_head->size);
+        free(ptr_with_head);
 	}
-    mem_head_t* ptr_with_head = AS_MEM_HEAD(ptr);
-    PyTruffle_FreeMemory(ptr_with_head->size);
-    free(ptr_with_head);
 }
 
 void* PyObject_Malloc(size_t size) {