pythongh-100926: Move ctype's pointers cache from _pointer_type_cache to StgInfo (pythonGH-131282)

Deprecate _pointer_type_cache and calling POINTER on a string.

Co-authored-by: neonene <[email protected]>
Co-authored-by: Jun Komoda <[email protected]>
Co-authored-by: Petr Viktorin <[email protected]>

Author: 4 people

Doc/library/ctypes.rst

@@ -2172,10 +2172,20 @@ Utility functions
 
 .. function:: POINTER(type, /)
 
-   Create and return a new ctypes pointer type. Pointer types are cached and
+   Create or return a ctypes pointer type. Pointer types are cached and
    reused internally, so calling this function repeatedly is cheap.
    *type* must be a ctypes type.
 
+   .. impl-detail::
+
+      The resulting pointer type is cached in the ``__pointer_type__``
+      attribute of *type*.
+      It is possible to set this attribute before the first call to
+      ``POINTER`` in order to set a custom pointer type.
+      However, doing this is discouraged: manually creating a suitable
+      pointer type is difficult without relying on implementation
+      details that may change in future Python versions.
+
 
 .. function:: pointer(obj, /)
 
@@ -2340,6 +2350,16 @@ Data types
       library. *name* is the name of the symbol that exports the data, *library*
       is the loaded shared library.
 
+   Common class variables of ctypes data types:
+
+   .. attribute:: __pointer_type__
+
+      The pointer type that was created by calling
+      :func:`POINTER` for corresponding ctypes data type. If a pointer type
+      was not yet created, the attribute is missing.
+
+      .. versionadded:: next
+
    Common instance variables of ctypes data types:
 
    .. attribute:: _b_base_

Doc/whatsnew/3.14.rst

@@ -807,11 +807,16 @@ ctypes
   loaded by the current process.
   (Contributed by Brian Ward in :gh:`119349`.)
 
+* Move :func:`ctypes.POINTER` types cache from a global internal cache
+  (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__`
+  attribute of the corresponding :mod:`ctypes` types.
+  This will stop the cache from growing without limits in some situations.
+  (Contributed by Sergey Miryanov in :gh:`100926`).
+
 * The :class:`ctypes.py_object` type now supports subscription,
   making it a :term:`generic type`.
   (Contributed by Brian Schubert in :gh:`132168`.)
 
-
 datetime
 --------
 
@@ -1679,6 +1684,13 @@ Deprecated
   :func:`codecs.open` is now deprecated. Use :func:`open` instead.
   (Contributed by Inada Naoki in :gh:`133036`.)
 
+* :mod:`ctypes`:
+  Calling :func:`ctypes.POINTER` on a string is deprecated.
+  Use :ref:`ctypes-incomplete-types` for self-referential structures.
+  Also, the internal ``ctypes._pointer_type_cache`` is deprecated.
+  See :func:`ctypes.POINTER` for updated implementation details.
+  (Contributed by Sergey Myrianov in :gh:`100926`.)
+
 * :mod:`functools`:
   Calling the Python implementation of :func:`functools.reduce` with *function*
   or *sequence* as keyword arguments is now deprecated.

Lib/ctypes/__init__.py

@@ -266,7 +266,72 @@ class c_void_p(_SimpleCData):
 class c_bool(_SimpleCData):
     _type_ = "?"
 
-from _ctypes import POINTER, pointer, _pointer_type_cache
+def POINTER(cls):
+    """Create and return a new ctypes pointer type.
+
+    Pointer types are cached and reused internally,
+    so calling this function repeatedly is cheap.
+    """
+    if cls is None:
+        return c_void_p
+    try:
+        return cls.__pointer_type__
+    except AttributeError:
+        pass
+    if isinstance(cls, str):
+        # handle old-style incomplete types (see test_ctypes.test_incomplete)
+        import warnings
+        warnings._deprecated("ctypes.POINTER with string", remove=(3, 19))
+        try:
+            return _pointer_type_cache_fallback[cls]
+        except KeyError:
+            result = type(f'LP_{cls}', (_Pointer,), {})
+            _pointer_type_cache_fallback[cls] = result
+            return result
+
+    # create pointer type and set __pointer_type__ for cls
+    return type(f'LP_{cls.__name__}', (_Pointer,), {'_type_': cls})
+
+def pointer(obj):
+    """Create a new pointer instance, pointing to 'obj'.
+
+    The returned object is of the type POINTER(type(obj)). Note that if you
+    just want to pass a pointer to an object to a foreign function call, you
+    should use byref(obj) which is much faster.
+    """
+    typ = POINTER(type(obj))
+    return typ(obj)
+
+class _PointerTypeCache:
+    def __setitem__(self, cls, pointer_type):
+        import warnings
+        warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19))
+        try:
+            cls.__pointer_type__ = pointer_type
+        except AttributeError:
+            _pointer_type_cache_fallback[cls] = pointer_type
+
+    def __getitem__(self, cls):
+        import warnings
+        warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19))
+        try:
+            return cls.__pointer_type__
+        except AttributeError:
+            return _pointer_type_cache_fallback[cls]
+
+    def get(self, cls, default=None):
+        import warnings
+        warnings._deprecated("ctypes._pointer_type_cache", remove=(3, 19))
+        try:
+            return cls.__pointer_type__
+        except AttributeError:
+            return _pointer_type_cache_fallback.get(cls, default)
+
+    def __contains__(self, cls):
+        return hasattr(cls, '__pointer_type__')
+
+_pointer_type_cache_fallback = {}
+_pointer_type_cache = _PointerTypeCache()
 
 class c_wchar_p(_SimpleCData):
     _type_ = "Z"
@@ -277,15 +342,14 @@ class c_wchar(_SimpleCData):
     _type_ = "u"
 
 def _reset_cache():
-    _pointer_type_cache.clear()
+    _pointer_type_cache_fallback.clear()
     _c_functype_cache.clear()
     if _os.name == "nt":
         _win_functype_cache.clear()
     # _SimpleCData.c_wchar_p_from_param
     POINTER(c_wchar).from_param = c_wchar_p.from_param
     # _SimpleCData.c_char_p_from_param
     POINTER(c_char).from_param = c_char_p.from_param
-    _pointer_type_cache[None] = c_void_p
 
 def create_unicode_buffer(init, size=None):
     """create_unicode_buffer(aString) -> character array
@@ -319,13 +383,7 @@ def create_unicode_buffer(init, size=None):
 def SetPointerType(pointer, cls):
     import warnings
     warnings._deprecated("ctypes.SetPointerType", remove=(3, 15))
-    if _pointer_type_cache.get(cls, None) is not None:
-        raise RuntimeError("This type already exists in the cache")
-    if id(pointer) not in _pointer_type_cache:
-        raise RuntimeError("What's this???")
     pointer.set_type(cls)
-    _pointer_type_cache[cls] = pointer
-    del _pointer_type_cache[id(pointer)]
 
 def ARRAY(typ, len):
     return typ * len

Lib/test/test_ctypes/test_byteswap.py

@@ -232,7 +232,6 @@ class TestStructure(parent):
                 self.assertEqual(len(data), sizeof(TestStructure))
                 ptr = POINTER(TestStructure)
                 s = cast(data, ptr)[0]
-                del ctypes._pointer_type_cache[TestStructure]
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)
 
@@ -371,7 +370,6 @@ class TestUnion(parent):
                 self.assertEqual(len(data), sizeof(TestUnion))
                 ptr = POINTER(TestUnion)
                 s = cast(data, ptr)[0]
-                del ctypes._pointer_type_cache[TestUnion]
                 self.assertEqual(s.point.x, 1)
                 self.assertEqual(s.point.y, 2)