gh-134761: Use deferred reference counting for threading concurrency primitives

Doc/library/sys.rst

@@ -290,6 +290,25 @@ always available. Unless explicitly noted otherwise, all variables are read-only
       This function is specific to CPython.  The exact output format is not
       defined here, and may change.
 
+.. function:: _defer_refcount(op)
+
+   Enable deferred reference counting on *op*, mitigating reference count
+   contention on :term:`free threaded <free threading>` builds of Python.
+
+   Return :const:`True` if deferred reference counting was enabled on *op*,
+   and :const:`False` otherwise.
+
+   .. versionadded:: next
+
+   .. impl-detail::
+
+      This function should be used for specialized purposes only.
+      It is not guaranteed to exist in all implementations of Python.
+
+   .. seealso::
+
+      :c:func:`PyUnstable_Object_EnableDeferredRefcount`
+
 
 .. data:: dllhandle
 

Lib/test/test_sys.py

@@ -1343,6 +1343,25 @@ def test_pystats(self):
     def test_disable_gil_abi(self):
         self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED)
 
+    @test.support.cpython_only
+    @unittest.skipUnless(hasattr(sys, '_defer_refcount'), "requires _defer_refcount()")
+    def test_defer_refcount(self):
+        _testinternalcapi = import_helper.import_module('_testinternalcapi')
+
+        class Test:
+            pass
+
+        ref = Test()
+        if support.Py_GIL_DISABLED:
+            self.assertTrue(sys._defer_refcount(ref))
+            self.assertTrue(_testinternalcapi.has_deferred_refcount(ref))
+            self.assertFalse(sys._defer_refcount(ref))
+            self.assertFalse(sys._defer_refcount(42))
+        else:
+            self.assertFalse(sys._defer_refcount(ref))
+            self.assertFalse(_testinternalcapi.has_deferred_refcount(ref))
+            self.assertFalse(sys._defer_refcount(42))
+
 
 @test.support.cpython_only
 class UnraisableHookTest(unittest.TestCase):

Lib/threading.py

@@ -73,6 +73,11 @@
 _profile_hook = None
 _trace_hook = None
 
+def _defer_refcount(op):
+    """Improve multithreaded scaling on the free-threade build."""
+    if hasattr(_sys, "_defer_refcount"):
+        _sys._defer_refcount(op)
+
 def setprofile(func):
     """Set a profile function for all threads started from the threading module.
 
@@ -298,6 +303,7 @@ def __init__(self, lock=None):
         if hasattr(lock, '_is_owned'):
             self._is_owned = lock._is_owned
         self._waiters = _deque()
+        _defer_refcount(self)
 
     def _at_fork_reinit(self):
         self._lock._at_fork_reinit()
@@ -466,6 +472,7 @@ def __init__(self, value=1):
             raise ValueError("semaphore initial value must be >= 0")
         self._cond = Condition(Lock())
         self._value = value
+        _defer_refcount(self)
 
     def __repr__(self):
         cls = self.__class__
@@ -595,6 +602,7 @@ class Event:
     def __init__(self):
         self._cond = Condition(Lock())
         self._flag = False
+        _defer_refcount(self)
 
     def __repr__(self):
         cls = self.__class__
@@ -700,6 +708,7 @@ def __init__(self, parties, action=None, timeout=None):
         self._parties = parties
         self._state = 0  # 0 filling, 1 draining, -1 resetting, -2 broken
         self._count = 0
+        _defer_refcount(self)
 
     def __repr__(self):
         cls = self.__class__

Misc/NEWS.d/next/Library/2025-05-26-19-49-16.gh-issue-134761.LeMefI.rst

@@ -0,0 +1,2 @@
+Improve performance when using :mod:`threading` primitives across multiple
+threads.

Modules/_threadmodule.c

@@ -950,6 +950,7 @@ lock_new_impl(PyTypeObject *type)
     if (self == NULL) {
         return NULL;
     }
+    _PyObject_SetDeferredRefcount((PyObject *)self);
     self->lock = (PyMutex){0};
     return (PyObject *)self;
 }
@@ -1221,6 +1222,7 @@ rlock_new_impl(PyTypeObject *type)
     if (self == NULL) {
         return NULL;
     }
+    _PyObject_SetDeferredRefcount((PyObject *)self);
     self->lock = (_PyRecursiveMutex){0};
     return (PyObject *) self;
 }

Python/sysmodule.c

@@ -15,6 +15,7 @@ Data members:
 */
 
 #include "Python.h"
+#include "object.h"
 #include "pycore_audit.h"         // _Py_AuditHookEntry
 #include "pycore_call.h"          // _PyObject_CallNoArgs()
 #include "pycore_ceval.h"         // _PyEval_SetAsyncGenFinalizer()
@@ -2653,6 +2654,23 @@ sys__is_gil_enabled_impl(PyObject *module)
 #endif
 }
 
+/*[clinic input]
+sys._defer_refcount -> bool
+
+  op: object
+  /
+
+Defer reference counting for the object, allowing for better scaling across multiple threads.
+
+This function should be used for specialized purposes only.
+[clinic start generated code]*/
+
+static int
+sys__defer_refcount_impl(PyObject *module, PyObject *op)
+/*[clinic end generated code: output=3b965122056085f5 input=a081971a76c49e64]*/
+{
+    return PyUnstable_Object_EnableDeferredRefcount(op);
+}
 
 #ifndef MS_WINDOWS
 static PerfMapState perf_map_state;
@@ -2834,6 +2852,7 @@ static PyMethodDef sys_methods[] = {
     SYS__GET_CPU_COUNT_CONFIG_METHODDEF
     SYS__IS_GIL_ENABLED_METHODDEF
     SYS__DUMP_TRACELETS_METHODDEF
+    SYS__DEFER_REFCOUNT_METHODDEF
     {NULL, NULL}  // sentinel
 };