Merge encukou seggestions from PR

Author: sergey-miryanov

Doc/whatsnew/3.14.rst

@@ -1580,6 +1580,14 @@ Deprecated
   as a single positional argument.
   (Contributed by Serhiy Storchaka in :gh:`109218`.)
 
+* :mod:`ctypes`:
+  Calling :func:`ctypes.POINTER` on a string is deprecated.
+  Use :ref:`ctypes-incomplete-types` for self-referential structures.
+  Also, ``ctypes._pointer_type_cache`` is deprecated as a courtesy to
+  existing users of this internal API.
+  See :func:`ctypes.POINTER` for updated implementation details.
+  (Contributed by Sergey Myrianov in :gh:`100926`.)
+
 * :mod:`functools`:
   Calling the Python implementation of :func:`functools.reduce` with *function*
   or *sequence* as keyword arguments is now deprecated.

Lib/ctypes/__init__.py

@@ -279,10 +279,15 @@ def POINTER(cls):
     except AttributeError:
         pass
     if isinstance(cls, str):
-        # handle old-style incomplete types
-        # in this case pointer type is not cached and calling this function
-        # repeatedly will give different result
-        return type(f'LP_{cls}', (_Pointer,), {})
+        # handle old-style incomplete types (see test_ctypes.test_incomplete)
+        import warnings
+        warnings._deprecated("ctypes.POINTER with string", remove=(3, 19))
+        try:
+            return _pointer_type_cache_fallback[cls]
+        except KeyError:
+            result = type(f'LP_{cls}', (_Pointer,), {})
+            _pointer_type_cache_fallback[cls] = result
+            return result
 
     # create pointer type and set __pointer_type__ for cls
     return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls})
@@ -297,22 +302,22 @@ def pointer(obj):
     typ = POINTER(type(obj))
     return typ(obj)
 
-class PointerTypeCache:
+class _PointerTypeCache:
     def __setitem__(self, cls, pointer_type):
         import warnings
         warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19))
         try:
             cls.__pointer_type__ = pointer_type
         except AttributeError:
-            pass
+            _pointer_type_cache_fallback[cls] = pointer_type
 
     def __getitem__(self, cls):
         import warnings
         warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19))
         try:
             return cls.__pointer_type__
         except AttributeError:
-            raise KeyError(cls)
+            return _pointer_type_cache_fallback[cls]
 
     def get(self, cls, default=None):
         import warnings
@@ -325,7 +330,8 @@ def get(self, cls, default=None):
     def __contains__(self, cls):
         return hasattr(cls, '__pointer_type__')
 
-_pointer_type_cache = PointerTypeCache()
+_pointer_type_cache_fallback = {}
+_pointer_type_cache = _PointerTypeCache()
 
 class c_wchar_p(_SimpleCData):
     _type_ = "Z"
@@ -376,9 +382,6 @@ def create_unicode_buffer(init, size=None):
 def SetPointerType(pointer, cls):
     import warnings
     warnings._deprecated("ctypes.SetPointerType", remove=(3, 15))
-    if _pointer_type_cache.get(cls, None) is not None:
-        raise RuntimeError("This type already exists in the cache")
-
     pointer.set_type(cls)
 
 def ARRAY(typ, len):

Lib/test/test_ctypes/test_c_simple_type_meta.py

@@ -10,9 +10,6 @@ def set_non_ctypes_pointer_type(cls, pointer_type):
     cls.__pointer_type__ = pointer_type
 
 class PyCSimpleTypeAsMetaclassTest(unittest.TestCase):
-    def tearDown(self):
-        ctypes._reset_cache()
-
     def test_creating_pointer_in_dunder_new_1(self):
         # Test metaclass whose instances are C types; when the type is
         # created it automatically creates a pointer type for itself.

Lib/test/test_ctypes/test_incomplete.py

@@ -3,14 +3,20 @@
 import warnings
 from ctypes import Structure, POINTER, pointer, c_char_p
 
+# String-based "incomplete pointers" wers implemented ctypes 0.6.3 (2003, when
+# ctypes was an external project). They made obsolete by the current
+# incomplete *types* (setting `_fields_` late) in 0.9.5 (2005).
+# ctypes was added to Python 2.5 (2006), without any mention in docs.
 
-# The incomplete pointer example from the tutorial
+# This tests incomplete pointer example from the old tutorial
+# (https://svn.python.org/projects/ctypes/tags/release_0_6_3/ctypes/docs/tutorial.stx)
 class TestSetPointerType(unittest.TestCase):
     def tearDown(self):
-        ctypes._reset_cache()
+        ctypes._pointer_type_cache_fallback.clear()
 
     def test_incomplete_example(self):
-        lpcell = POINTER("cell")
+        with self.assertWarns(DeprecationWarning):
+            lpcell = POINTER("cell")
         class cell(Structure):
             _fields_ = [("name", c_char_p),
                         ("next", lpcell)]
@@ -38,7 +44,8 @@ class cell(Structure):
         self.assertEqual(result, [b"foo", b"bar"] * 4)
 
     def test_deprecation(self):
-        lpcell = POINTER("cell")
+        with self.assertWarns(DeprecationWarning):
+            lpcell = POINTER("cell")
         class cell(Structure):
             _fields_ = [("name", c_char_p),
                         ("next", lpcell)]

Lib/test/test_ctypes/test_pointers.py

@@ -10,7 +10,7 @@
                     c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint,
                     c_long, c_ulong, c_longlong, c_ulonglong,
                     c_float, c_double)
-from ctypes import _pointer_type_cache
+from ctypes import _pointer_type_cache, _pointer_type_cache_fallback
 from test.support import import_helper
 from weakref import WeakSet
 _ctypes_test = import_helper.import_module("_ctypes_test")
@@ -25,6 +25,9 @@
 
 
 class PointersTestCase(unittest.TestCase):
+    def tearDown(self):
+        _pointer_type_cache_fallback.clear()
+
     def test_inheritance_hierarchy(self):
         self.assertEqual(_Pointer.mro(), [_Pointer, _CData, object])
 
@@ -246,7 +249,8 @@ def test_pointer_type_name(self):
 
     def test_pointer_type_str_name(self):
         large_string = 'T' * 2 ** 25
-        P = POINTER(large_string)
+        with self.assertWarns(DeprecationWarning):
+            P = POINTER(large_string)
         self.assertTrue(P)
 
     def test_abstract(self):
@@ -267,14 +271,17 @@ def test_pointer_types_equal(self):
         self.assertIs(type(p1), t1)
         self.assertIs(type(p2), t1)
 
-    def test_incomplete_pointer_types_not_equal(self):
-        t1 = POINTER("LP_C")
-        t2 = POINTER("LP_C")
+    def test_incomplete_pointer_types_still_equal(self):
+        with self.assertWarns(DeprecationWarning):
+            t1 = POINTER("LP_C")
+        with self.assertWarns(DeprecationWarning):
+            t2 = POINTER("LP_C")
 
-        self.assertIsNot(t1, t2)
+        self.assertIs(t1, t2)
 
     def test_incomplete_pointer_types_cannot_instantiate(self):
-        t1 = POINTER("LP_C")
+        with self.assertWarns(DeprecationWarning):
+            t1 = POINTER("LP_C")
         with self.assertRaisesRegex(TypeError, "has no _type_"):
             t1()
 
@@ -288,15 +295,29 @@ def test_pointer_set_type_twice(self):
         self.assertIs(t1._type_, c_int)
 
     def test_pointer_set_wrong_type(self):
-        class C(c_int):
-            pass
-
-        t1 = POINTER(c_int)
-        with self.assertRaisesRegex(TypeError, "pointer type already set"):
+        int_ptr = POINTER(c_int)
+        float_ptr = POINTER(float_ptr)
+        try:
+            class C(c_int):
+                pass
+
+            t1 = POINTER(c_int)
+            t2 = POINTER(c_float)
             t1.set_type(c_float)
+            self.assertEqual(t1(c_float(1.5))[0], 1.5)
+            self.assertIs(c_int._type_, c_float)
+            self.assertIs(c_int.__pointer_type__, t1)
+            self.assertIs(c_float.__pointer_type__, float_ptr)
 
-        with self.assertRaisesRegex(TypeError, "cls type already set"):
             t1.set_type(C)
+            self.assertEqual(t1(C(123))[0].value, 123)
+            self.assertIs(c_int.__pointer_type__, t1)
+            self.assertIs(c_float.__pointer_type__, float_ptr)
+        finally:
+            POINTER(c_int).set_type(c_int)
+        self.assertIs(POINTER(c_int), int_ptr)
+        self.assertIs(POINTER(c_int)._type_, c_int)
+        self.assertIs(c_int.__pointer_type__, int_ptr)
 
     def test_pointer_not_ctypes_type(self):
         with self.assertRaisesRegex(TypeError, "must have storage info"):
@@ -326,6 +347,34 @@ class Cls(Structure):
         p = POINTER(Cls)
         self.assertIs(Cls.__pointer_type__, p)
 
+    def test_arbitrary_pointer_type_attribute(self):
+        class Cls(Structure):
+            _fields_ = (
+                ('a', c_int),
+                ('b', c_float),
+            )
+
+        garbage = 'garbage'
+
+        P = POINTER(Cls)
+        self.assertIs(Cls.__pointer_type__, P)
+        Cls.__pointer_type__ = garbage
+        self.assertIs(Cls.__pointer_type__, garbage)
+        self.assertIs(POINTER(Cls), garbage)
+        self.assertIs(P._type_, Cls)
+
+        instance = Cls(1, 2.0)
+        pointer = P(instance)
+        self.assertEqual(pointer[0].a, 1)
+        self.assertEqual(pointer[0].b, 2)
+
+        del Cls.__pointer_type__
+
+        NewP = POINTER(Cls)
+        self.assertIsNot(NewP, P)
+        self.assertIs(Cls.__pointer_type__, NewP)
+        self.assertIs(P._type_, Cls)
+
     def test_pointer_types_factory(self):
         """Shouldn't leak"""
         def factory():
@@ -357,21 +406,31 @@ class Cls(Structure):
 
 class PointerTypeCacheTestCase(unittest.TestCase):
     # dummy tests to check warnings and base behavior
+    def tearDown(self):
+        _pointer_type_cache_fallback.clear()
 
     def test_deprecated_cache_with_not_ctypes_type(self):
         class C:
             pass
 
-        P = POINTER("C")
         with self.assertWarns(DeprecationWarning):
-            _pointer_type_cache[C] = P
+            P = POINTER("C")
 
+        with self.assertWarns(DeprecationWarning):
+            self.assertIs(_pointer_type_cache["C"], P)
+
+        with self.assertWarns(DeprecationWarning):
+            _pointer_type_cache[C] = P
         self.assertIs(C.__pointer_type__, P)
         with self.assertWarns(DeprecationWarning):
             self.assertIs(_pointer_type_cache[C], P)
 
+    def test_deprecated_cache_with_ints(self):
+        with self.assertWarns(DeprecationWarning):
+            _pointer_type_cache[123] = 456
+
         with self.assertWarns(DeprecationWarning):
-            self.assertIs(_pointer_type_cache.get(C), P)
+            self.assertEqual(_pointer_type_cache[123], 456)
 
     def test_deprecated_cache_with_ctypes_type(self):
         class C(Structure):
@@ -380,21 +439,20 @@ class C(Structure):
                         ("c", c_int)]
 
         P1 = POINTER(C)
-        P2 = POINTER("C")
         with self.assertWarns(DeprecationWarning):
-            _pointer_type_cache[C] = P1
+            P2 = POINTER("C")
 
         with self.assertWarns(DeprecationWarning):
-            _pointer_type_cache[C] = P2 # silently do nothing
+            _pointer_type_cache[C] = P2
 
-        self.assertIs(C.__pointer_type__, P1)
-        self.assertIsNot(C.__pointer_type__, P2)
+        self.assertIs(C.__pointer_type__, P2)
+        self.assertIsNot(C.__pointer_type__, P1)
 
         with self.assertWarns(DeprecationWarning):
-            self.assertIs(_pointer_type_cache[C], P1)
+            self.assertIs(_pointer_type_cache[C], P2)
 
         with self.assertWarns(DeprecationWarning):
-            self.assertIs(_pointer_type_cache.get(C), P1)
+            self.assertIs(_pointer_type_cache.get(C), P2)
 
     def test_get_not_registered(self):
         with self.assertWarns(DeprecationWarning):

Modules/_ctypes/_ctypes.c

@@ -599,6 +599,23 @@ ctype_get_pointer_type(PyObject *self, void *Py_UNUSED(ignored))
     return NULL;
 }
 
+static int
+ctype_set_pointer_type(PyObject *self, PyObject *tp, void *Py_UNUSED(ignored))
+{
+    ctypes_state *st = get_module_state_by_def(Py_TYPE(self));
+    StgInfo *info;
+    if (PyStgInfo_FromType(st, self, &info) < 0) {
+        return -1;
+    }
+    if (!info) {
+        PyErr_Format(PyExc_TypeError, "%R must have storage info", self);
+        return -1;
+    }
+
+    Py_XSETREF(info->pointer_type, Py_XNewRef(tp));
+    return 0;
+}
+
 static PyObject *
 CType_Type_repeat(PyObject *self, Py_ssize_t length);
 
@@ -609,7 +626,8 @@ static PyMethodDef ctype_methods[] = {
 };
 
 static PyGetSetDef ctype_getsets[] = {
-    { "__pointer_type__", ctype_get_pointer_type, NULL, "pointer type", NULL },
+    { "__pointer_type__", ctype_get_pointer_type, ctype_set_pointer_type,
+      "pointer type", NULL },
     { NULL, NULL }
 };
 
@@ -1235,25 +1253,9 @@ PyCPointerType_SetProto(ctypes_state *st, PyObject *self, StgInfo *stginfo, PyOb
         PyErr_Format(PyExc_TypeError, "%R must have storage info", proto);
         return -1;
     }
-    if (info->pointer_type && info->pointer_type != self) {
-        PyErr_Format(PyExc_TypeError,
-            "pointer type already set: old=%R, new=%R",
-            info->pointer_type, self);
-        return -1;
-    }
-    if (stginfo->proto && stginfo->proto != proto) {
-        PyErr_Format(PyExc_TypeError,
-            "cls type already set: old=%R, new=%R",
-            stginfo->proto, proto);
-        return -1;
-    }
-
-    if (!stginfo->proto) {
-        stginfo->proto = Py_NewRef(proto);
-    }
-
-    if (!info->pointer_type) {
-        info->pointer_type = Py_NewRef(self);
+    Py_XSETREF(stginfo->proto, Py_NewRef(proto));
+    if (info->pointer_type == NULL) {
+        Py_XSETREF(info->pointer_type, Py_NewRef(self));
     }
     return 0;
 }

Modules/_ctypes/ctypes.h

@@ -388,7 +388,8 @@ typedef struct {
     PyObject *converters;       /* tuple([t.from_param for t in argtypes]) */
     PyObject *restype;          /* CDataObject or NULL */
     PyObject *checker;
-    PyObject *pointer_type;
+    PyObject *pointer_type;     /* __pointer_type__ attribute;
+                                   arbitrary object or NULL */
     PyObject *module;
     int flags;                  /* calling convention and such */
 #ifdef Py_GIL_DISABLED