gh-127870: Fix ctypes _as_parameter_ handling

Detect recursive calls in ctypes _as_parameter_ handling: add
Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() calls.

Author: vstinner

Lib/test/test_ctypes/test_as_parameter.py

@@ -5,6 +5,7 @@
                     c_short, c_int, c_long, c_longlong,
                     c_byte, c_wchar, c_float, c_double,
                     ArgumentError)
+import ctypes
 from test.support import import_helper
 _ctypes_test = import_helper.import_module("_ctypes_test")
 
@@ -198,8 +199,16 @@ class A:
 
         a = A()
         a._as_parameter_ = a
-        with self.assertRaises(RecursionError):
-            c_int.from_param(a)
+        for c_type in (
+            ctypes.c_wchar_p,
+            ctypes.c_char_p,
+            ctypes.c_void_p,
+            ctypes.c_int,  # PyCSimpleType
+            POINT,  # CDataType
+        ):
+            with self.subTest(c_type=c_type):
+                with self.assertRaises(RecursionError):
+                    c_type.from_param(a)
 
 
 class AsParamWrapper:

Misc/NEWS.d/next/Library/2024-12-12-16-59-42.gh-issue-127870._NFG-3.rst

@@ -0,0 +1,2 @@
+Detect recursive calls in ctypes ``_as_parameter_`` handling.
+Patch by Victor Stinner.

Modules/_ctypes/_ctypes.c

@@ -1052,8 +1052,13 @@ CDataType_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value)
         return NULL;
     }
     if (as_parameter) {
+        if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) {
+            Py_DECREF(as_parameter);
+            return NULL;
+        }
         value = CDataType_from_param_impl(type, cls, as_parameter);
         Py_DECREF(as_parameter);
+        _Py_LeaveRecursiveCall();
         return value;
     }
     PyErr_Format(PyExc_TypeError,
@@ -1843,8 +1848,13 @@ c_wchar_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value)
         return NULL;
     }
     if (as_parameter) {
+        if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) {
+            Py_DECREF(as_parameter);
+            return NULL;
+        }
         value = c_wchar_p_from_param_impl(type, cls, as_parameter);
         Py_DECREF(as_parameter);
+        _Py_LeaveRecursiveCall();
         return value;
     }
     PyErr_Format(PyExc_TypeError,
@@ -1927,8 +1937,13 @@ c_char_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value)
         return NULL;
     }
     if (as_parameter) {
+        if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) {
+            Py_DECREF(as_parameter);
+            return NULL;
+        }
         value = c_char_p_from_param_impl(type, cls, as_parameter);
         Py_DECREF(as_parameter);
+        _Py_LeaveRecursiveCall();
         return value;
     }
     PyErr_Format(PyExc_TypeError,
@@ -2079,8 +2094,13 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value)
         return NULL;
     }
     if (as_parameter) {
+        if (_Py_EnterRecursiveCall(" while processing _as_parameter_")) {
+            Py_DECREF(as_parameter);
+            return NULL;
+        }
         value = c_void_p_from_param_impl(type, cls, as_parameter);
         Py_DECREF(as_parameter);
+        _Py_LeaveRecursiveCall();
         return value;
     }
     PyErr_Format(PyExc_TypeError,
@@ -2447,9 +2467,9 @@ PyCSimpleType_from_param_impl(PyObject *type, PyTypeObject *cls,
             return NULL;
         }
         value = PyCSimpleType_from_param_impl(type, cls, as_parameter);
-        _Py_LeaveRecursiveCall();
         Py_DECREF(as_parameter);
         Py_XDECREF(exc);
+        _Py_LeaveRecursiveCall();
         return value;
     }
     if (exc) {