[3.14] pythongh-133296: Publicly expose critical section API that accepts PyMutex (pythongh-135899) (python#136969)

Co-authored-by: Nathan Goldbaum <[email protected]>
Co-authored-by: Hugo van Kemenade <[email protected]>

Author: 3 people

Doc/c-api/init.rst

@@ -2458,6 +2458,12 @@ is resumed, and its locks reacquired.  This means the critical section API
 provides weaker guarantees than traditional locks -- they are useful because
 their behavior is similar to the :term:`GIL`.
 
+Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also
+available. Use these variants to start a critical section in a situation where
+there is no :c:type:`PyObject` -- for example, when working with a C type that
+does not extend or wrap :c:type:`PyObject` but still needs to call into the C
+API in a manner that might lead to deadlocks.
+
 The functions and structs used by the macros are exposed for cases
 where C macros are not available. They should only be used as in the
 given macro expansions. Note that the sizes and contents of the structures may
@@ -2503,6 +2509,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`.
 
    .. versionadded:: 3.13
 
+.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m)
+
+   Locks the mutex *m* and begins a critical section.
+
+   In the free-threaded build, this macro expands to::
+
+     {
+          PyCriticalSection _py_cs;
+          PyCriticalSection_BeginMutex(&_py_cs, m)
+
+   Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for
+   the argument of the macro - it must be a :c:type:`PyMutex` pointer.
+
+   On the default build, this macro expands to ``{``.
+
+   .. versionadded:: 3.14
+
 .. c:macro:: Py_END_CRITICAL_SECTION()
 
    Ends the critical section and releases the per-object lock.
@@ -2532,6 +2555,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`.
 
    .. versionadded:: 3.13
 
+.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2)
+
+   Locks the mutexes *m1* and *m2* and begins a critical section.
+
+   In the free-threaded build, this macro expands to::
+
+     {
+          PyCriticalSection2 _py_cs2;
+          PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2)
+
+   Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for
+   the arguments of the macro - they must be :c:type:`PyMutex` pointers.
+
+   On the default build, this macro expands to ``{``.
+
+   .. versionadded:: 3.14
+
 .. c:macro:: Py_END_CRITICAL_SECTION2()
 
    Ends the critical section and releases the per-object locks.

Include/cpython/critical_section.h

@@ -73,22 +73,32 @@ typedef struct PyCriticalSection2 PyCriticalSection2;
 PyAPI_FUNC(void)
 PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op);
 
+PyAPI_FUNC(void)
+PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m);
+
 PyAPI_FUNC(void)
 PyCriticalSection_End(PyCriticalSection *c);
 
 PyAPI_FUNC(void)
 PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b);
 
+PyAPI_FUNC(void)
+PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2);
+
 PyAPI_FUNC(void)
 PyCriticalSection2_End(PyCriticalSection2 *c);
 
 #ifndef Py_GIL_DISABLED
 # define Py_BEGIN_CRITICAL_SECTION(op)      \
     {
+# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex)    \
+    {
 # define Py_END_CRITICAL_SECTION()          \
     }
 # define Py_BEGIN_CRITICAL_SECTION2(a, b)   \
     {
+# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2)  \
+    {
 # define Py_END_CRITICAL_SECTION2()         \
     }
 #else /* !Py_GIL_DISABLED */
@@ -118,6 +128,11 @@ struct PyCriticalSection2 {
         PyCriticalSection _py_cs;                                       \
         PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op))
 
+# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex)                         \
+    {                                                                   \
+        PyCriticalSection _py_cs;                                       \
+        PyCriticalSection_BeginMutex(&_py_cs, mutex)
+
 # define Py_END_CRITICAL_SECTION()                                      \
         PyCriticalSection_End(&_py_cs);                                 \
     }
@@ -127,6 +142,11 @@ struct PyCriticalSection2 {
         PyCriticalSection2 _py_cs2;                                     \
         PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b))
 
+# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2)                       \
+    {                                                                   \
+        PyCriticalSection2 _py_cs2;                                     \
+        PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2)
+
 # define Py_END_CRITICAL_SECTION2()                                     \
         PyCriticalSection2_End(&_py_cs2);                               \
     }

Include/internal/pycore_critical_section.h

@@ -21,16 +21,6 @@ extern "C" {
 #define _Py_CRITICAL_SECTION_MASK           0x3
 
 #ifdef Py_GIL_DISABLED
-# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex)                           \
-    {                                                                   \
-        PyCriticalSection _py_cs;                                       \
-        _PyCriticalSection_BeginMutex(&_py_cs, mutex)
-
-# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2)                         \
-    {                                                                   \
-        PyCriticalSection2 _py_cs2;                                     \
-        _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2)
-
 // Specialized version of critical section locking to safely use
 // PySequence_Fast APIs without the GIL. For performance, the argument *to*
 // PySequence_Fast() is provided to the macro, not the *result* of
@@ -75,8 +65,6 @@ extern "C" {
 
 #else  /* !Py_GIL_DISABLED */
 // The critical section APIs are no-ops with the GIL.
-# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) {
-# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) {
 # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) {
 # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() }
 # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex)
@@ -119,6 +107,7 @@ _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)
         _PyCriticalSection_BeginSlow(c, m);
     }
 }
+#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex
 
 static inline void
 _PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
@@ -194,6 +183,7 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
         _PyCriticalSection2_BeginSlow(c, m1, m2, 0);
     }
 }
+#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex
 
 static inline void
 _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)

Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst

@@ -0,0 +1,3 @@
+New variants for the critical section API that accept one or two
+:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now
+public in the non-limited C API.

Modules/_ctypes/ctypes.h

@@ -419,7 +419,7 @@ typedef struct {
     visible to other threads before the `dict_final` bit is set.
 */
 
-#define STGINFO_LOCK(stginfo)   Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex)
+#define STGINFO_LOCK(stginfo)   Py_BEGIN_CRITICAL_SECTION_MUTEX(&(stginfo)->mutex)
 #define STGINFO_UNLOCK()        Py_END_CRITICAL_SECTION()
 
 static inline uint8_t

Modules/_testcapimodule.c

@@ -2419,6 +2419,16 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args))
     Py_BEGIN_CRITICAL_SECTION2(module, module);
     Py_END_CRITICAL_SECTION2();
 
+#ifdef Py_GIL_DISABLED
+    // avoid unused variable compiler warning on GIL-enabled build
+    PyMutex mut = {0};
+    Py_BEGIN_CRITICAL_SECTION_MUTEX(&mut);
+    Py_END_CRITICAL_SECTION();
+
+    Py_BEGIN_CRITICAL_SECTION2_MUTEX(&mut, &mut);
+    Py_END_CRITICAL_SECTION2();
+#endif
+
     Py_RETURN_NONE;
 }
 

Objects/typeobject.c

@@ -65,11 +65,11 @@ class object "PyObject *" "&PyBaseObject_Type"
 // be released and reacquired during a subclass update if there's contention
 // on the subclass lock.
 #define TYPE_LOCK &PyInterpreterState_Get()->types.mutex
-#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK)
+#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK)
 #define END_TYPE_LOCK() Py_END_CRITICAL_SECTION()
 
 #define BEGIN_TYPE_DICT_LOCK(d) \
-    Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex)
+    Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex)
 
 #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2()
 

Python/critical_section.c

@@ -130,6 +130,15 @@ PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op)
 #endif
 }
 
+#undef PyCriticalSection_BeginMutex
+void
+PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m)
+{
+#ifdef Py_GIL_DISABLED
+    _PyCriticalSection_BeginMutex(c, m);
+#endif
+}
+
 #undef PyCriticalSection_End
 void
 PyCriticalSection_End(PyCriticalSection *c)
@@ -148,6 +157,15 @@ PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b)
 #endif
 }
 
+#undef PyCriticalSection2_BeginMutex
+void
+PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2)
+{
+#ifdef Py_GIL_DISABLED
+    _PyCriticalSection2_BeginMutex(c, m1, m2);
+#endif
+}
+
 #undef PyCriticalSection2_End
 void
 PyCriticalSection2_End(PyCriticalSection2 *c)