[GR-52707][GR-52721] Update torchvision

PullRequest: graalpython/3251

Author: msimacek

graalpython/com.oracle.graal.python.cext/include/internal/pycore_object.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2022, 2023, Oracle and/or its affiliates.
+/* Copyright (c) 2022, 2024, Oracle and/or its affiliates.
  * Copyright (C) 1996-2022 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -203,17 +203,14 @@ _PyObject_IS_GC(PyObject *obj)
 // Fast inlined version of PyType_IS_GC()
 #define _PyType_IS_GC(t) _PyType_HasFeature((t), Py_TPFLAGS_HAVE_GC)
 
-// GraalPy change: we don't want to pull in CPython GC implementation details
-/*
 static inline size_t
 _PyType_PreHeaderSize(PyTypeObject *tp)
 {
-    return _PyType_IS_GC(tp) * sizeof(PyGC_Head) +
-        _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * 2 * sizeof(PyObject *);
+    // GraalPy change: remove CPython's GC header; also we put only one pointer for dict, we don't store it inlined
+    return _PyType_HasFeature(tp, Py_TPFLAGS_MANAGED_DICT) * sizeof(PyObject *);
 }
 
 void _PyObject_GC_Link(PyObject *op);
-*/
 
 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
 extern int _Py_CheckSlotResult(
@@ -240,14 +237,16 @@ PyObject * _PyObject_GetInstanceAttribute(PyObject *obj, PyDictValues *values,
 
 static inline PyDictValues **_PyObject_ValuesPointer(PyObject *obj)
 {
-    assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    return ((PyDictValues **)obj)-4;
+    // GraalPy change: we don't have inlined managed dict values
+    assert(0);
+    return NULL;
 }
 
 static inline PyObject **_PyObject_ManagedDictPointer(PyObject *obj)
 {
     assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
-    return ((PyObject **)obj)-3;
+    // GraalPy change: ours is at a different offset
+    return ((PyObject **)obj)-1;
 }
 
 #define MANAGED_DICT_OFFSET (((int)sizeof(PyObject *))*-3)

graalpython/com.oracle.graal.python.cext/src/dictobject.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * The Universal Permissive License (UPL), Version 1.0
@@ -40,6 +40,8 @@
  */
 #include "capi.h"
 
+#include "pycore_object.h"        // _PyType_CheckConsistency(), _Py_FatalRefcountError()
+
 
 PyObject* _PyDict_NewPresized(Py_ssize_t minused) {
     /* we ignore requests to capacity for now */
@@ -110,53 +112,67 @@ int PyDict_DelItemString(PyObject *d, const char *key) {
 	CALL_WITH_STRING(key, int, -1, GraalPyDict_DelItem, d, string);
 }
 
-PyAPI_FUNC(PyObject*) _PyObject_GenericGetDict(PyObject* obj) {
-    PyObject** dictptr = _PyObject_GetDictPtr(obj);
+PyObject *
+PyObject_GenericGetDict(PyObject *obj, void *context)
+{
+    PyObject *dict;
+    // GraalPy change: we don't have inlined values in managed dict
+    PyObject **dictptr = _PyObject_GetDictPtr(obj);
     if (dictptr == NULL) {
+        PyErr_SetString(PyExc_AttributeError,
+                        "This object has no __dict__");
         return NULL;
     }
-    PyObject* dict = *dictptr;
+    dict = *dictptr;
     if (dict == NULL) {
+        // GraalPy change: we don't have CPython's cache
         *dictptr = dict = PyDict_New();
     }
     Py_XINCREF(dict);
     return dict;
 }
 
-PyObject* PyObject_GenericGetDict(PyObject* obj, void* context) {
-    PyObject* d = _PyObject_GenericGetDict(obj);
-    if (d == NULL) {
-        PyErr_SetString(PyExc_AttributeError, "This object has no __dict__");
-    }
-    return d;
-}
-
-PyObject** _PyObject_GetDictPtr(PyObject* obj) {
+PyObject **
+_PyObject_DictPointer(PyObject *obj)
+{
     Py_ssize_t dictoffset;
-
-    // relies on the fact that 'tp_dictoffset' is in sync with the corresponding managed class !
     PyTypeObject *tp = Py_TYPE(obj);
 
+    if (tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) {
+        return _PyObject_ManagedDictPointer(obj);
+    }
     dictoffset = tp->tp_dictoffset;
-    if (dictoffset == 0) {
+    if (dictoffset == 0)
         return NULL;
-    }
     if (dictoffset < 0) {
-        Py_ssize_t nitems = ((PyVarObject *)obj)->ob_size;
-        if (nitems < 0) {
-            nitems = -nitems;
+        Py_ssize_t tsize = Py_SIZE(obj);
+        if (tsize < 0) {
+            tsize = -tsize;
         }
+        size_t size = _PyObject_VAR_SIZE(tp, tsize);
+        assert(size <= (size_t)PY_SSIZE_T_MAX);
+        dictoffset += (Py_ssize_t)size;
 
-        size_t size = tp->tp_basicsize + nitems * tp->tp_itemsize;
-        if (size % SIZEOF_VOID_P != 0) {
-            // round to full pointer boundary
-            size += SIZEOF_VOID_P - (size % SIZEOF_VOID_P);
-        }
-        dictoffset += (long)size;
+        _PyObject_ASSERT(obj, dictoffset > 0);
+        _PyObject_ASSERT(obj, dictoffset % SIZEOF_VOID_P == 0);
     }
     return (PyObject **) ((char *)obj + dictoffset);
 }
 
+/* Helper to get a pointer to an object's __dict__ slot, if any.
+ * Creates the dict from inline attributes if necessary.
+ * Does not set an exception.
+ *
+ * Note that the tp_dictoffset docs used to recommend this function,
+ * so it should be treated as part of the public API.
+ */
+PyObject **
+_PyObject_GetDictPtr(PyObject *obj)
+{
+    // GraalPy change: we don't have inlined managed dict values, so just use the common path
+    return _PyObject_DictPointer(obj);
+}
+
 /* Taken from CPython */
 int
 _PyDict_ContainsId(PyObject *op, struct _Py_Identifier *key)

graalpython/com.oracle.graal.python.cext/src/object.c

@@ -916,8 +916,9 @@ PyVarObject* _PyObject_GC_NewVar(PyTypeObject *tp, Py_ssize_t nitems) {
     return _PyObject_NewVar(tp, nitems);
 }
 
-void PyObject_GC_Del(void *tp) {
-    PyObject_Free(tp);
+void PyObject_GC_Del(void *op) {
+    size_t presize = _PyType_PreHeaderSize(((PyObject *)op)->ob_type);
+    PyObject_Free(((char *)op)-presize);
 }
 
 void PyObject_GC_Track(void* a) {

graalpython/com.oracle.graal.python.cext/src/typeobject.c

@@ -1118,12 +1118,16 @@ _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems)
     const size_t size = _PyObject_VAR_SIZE(type, nitems+1);
     /* note that we need to add one, for the sentinel */
 
-    // GraalPy change: remove the GC header
-    char *alloc = PyObject_Malloc(size);
+    const size_t presize = _PyType_PreHeaderSize(type);
+    char *alloc = PyObject_Malloc(size + presize);
     if (alloc  == NULL) {
         return PyErr_NoMemory();
     }
-    obj = (PyObject *)alloc;
+    obj = (PyObject *)(alloc + presize);
+    if (presize) {
+        // GraalPy change: different header layout, no GC link
+        ((PyObject **)alloc)[0] = NULL;
+    }
     memset(obj, '\0', size);
 
     if (type->tp_itemsize == 0) {

graalpython/com.oracle.graal.python.test/src/tests/cpyext/__init__.py

@@ -754,13 +754,18 @@ def CPyExtHeapType(name, bases=(object), code='', slots=None, **kwargs):
 
     {includes}
 
+    typedef struct {{
+        {struct_base}
+        {cmembers}
+    }} {name}Object;
+
     {code}
 
     PyType_Slot slots[] = {{
         {slots}
     }};
 
-    PyType_Spec spec = {{ "{name}Type", sizeof(PyHeapTypeObject), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots }};
+    PyType_Spec spec = {{ "{name}Type", sizeof({name}Object), 0, Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, slots }};
 
     static PyObject* create(PyObject* unused, PyObject* bases) {{
         {ready_code}
@@ -798,6 +803,8 @@ def CPyExtHeapType(name, bases=(object), code='', slots=None, **kwargs):
     kwargs["code"] = code
     kwargs["slots"] = '{0}' if slots is None else ',\n'.join(slots + ['{0}'])
     kwargs.setdefault("includes", "")
+    kwargs.setdefault("struct_base", "PyObject_HEAD")
+    kwargs.setdefault("cmembers", "")
     kwargs.setdefault("ready_code", "")
     kwargs.setdefault("post_ready_code", "")
     code = UnseenFormatter().format(template, **kwargs)

graalpython/com.oracle.graal.python.test/src/tests/cpyext/test_object_dict.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+from tests.cpyext import CPyExtType, CPyExtHeapType, assert_raises
+
+NativeTypeWithoutDict = CPyExtType(
+    name='NativeTypeWithoutDict',
+)
+
+NativeTypeWithDict = CPyExtType(
+    name='NativeTypeWithDict',
+    cmembers='PyObject* dict;',
+    tp_dictoffset='offsetof(NativeTypeWithDictObject, dict)',
+    tp_getset='{"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict}',
+)
+
+NativeHeapTypeWithoutDict = CPyExtHeapType(
+    name='NativeTypeWithManagedDict',
+)
+
+NativeHeapTypeWithDict = CPyExtHeapType(
+    name='NativeHeapTypeWithManagedDict',
+    cmembers='PyObject* dict;',
+    code='''
+        static PyMemberDef memberlist[] = {
+            {"__dictoffset__", T_PYSSIZET, offsetof(NativeHeapTypeWithManagedDictObject, dict), READONLY},
+            {NULL}
+        };
+        static PyGetSetDef getsetlist[] = {
+            {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict},
+            {NULL}
+        };
+    ''',
+    slots=[
+        '{Py_tp_members, memberlist}',
+        '{Py_tp_getset, getsetlist}',
+    ],
+)
+
+
+# TODO it would be great if we could test creating heap types with Py_TPFLAGS_MANAGED_DICT, because pybind11 does that,
+# but there's no way to do that without abusing abusing implementations details
+
+
+class NativeSubtypeWithDict(NativeTypeWithDict):
+    pass
+
+
+class NativeSubtypeWithAddedDict(NativeTypeWithoutDict):
+    pass
+
+
+class TestObjectDict:
+    @staticmethod
+    def assert_has_no_dict(obj):
+        assert_raises(AttributeError, setattr, obj, 'foo', 1)
+        assert_raises(AttributeError, getattr, obj, '__dict__')
+
+    @staticmethod
+    def assert_has_working_dict(obj):
+        obj.foo = 1
+        assert obj.foo == 1
+        assert obj.__dict__ == {'foo': 1}
+
+    def test_no_dict(self):
+        self.assert_has_no_dict(NativeTypeWithoutDict())
+        self.assert_has_no_dict(NativeHeapTypeWithoutDict())
+
+    def test_dict(self):
+        self.assert_has_working_dict(NativeTypeWithDict())
+        self.assert_has_working_dict(NativeHeapTypeWithDict())
+        self.assert_has_working_dict(NativeSubtypeWithDict())
+        self.assert_has_working_dict(NativeSubtypeWithAddedDict())

graalpython/com.oracle.graal.python.test/src/tests/package/moduleA.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -40,3 +40,6 @@
 """MODULE A DOC"""
 
 print("module A")
+
+publicProperty = 1
+_privateProperty = 2

graalpython/com.oracle.graal.python.test/src/tests/package/moduleY.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -41,3 +41,8 @@
 import math
 print("cos(100) = "+str(math.cos(100)))
 from .subpackage1 import moduleX
+
+__all__ = ['publicProperty', '_privateProperty']
+publicProperty = 1
+_privateProperty = 2
+unlistedProperty = 3

graalpython/com.oracle.graal.python.test/src/tests/test_imports.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2018, 2021, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -93,6 +93,21 @@ def test_import_star_has_to_be_module():
         assert False
 
 
+def test_import_star_no_all():
+    g = {}
+    exec("from package.moduleA import *", g)
+    assert 'publicProperty' in g
+    assert '_privateProperty' not in g
+
+
+def test_import_star_all():
+    g = {}
+    exec("from package.moduleY import *", g)
+    assert 'publicProperty' in g
+    assert '_privateProperty' in g
+    assert 'unlistedProperty' not in g
+
+
 def test_import_as_local():
     from sys import path
     from sys import path as mypath