gh-136003: Execute pre-finalization callbacks in a loop

Include/internal/pycore_ceval.h

@@ -43,7 +43,7 @@ PyAPI_FUNC(int) _PyEval_MakePendingCalls(PyThreadState *);
 #  define Py_DEFAULT_RECURSION_LIMIT 1000
 #endif
 
-extern void _Py_FinishPendingCalls(PyThreadState *tstate);
+extern int _Py_FinishPendingCalls(PyThreadState *tstate);
 extern void _PyEval_InitState(PyInterpreterState *);
 extern void _PyEval_SignalReceived(void);
 

Include/internal/pycore_pylifecycle.h

@@ -85,7 +85,7 @@ extern void _PyErr_DisplayException(PyObject *file, PyObject *exc);
 
 extern void _PyThreadState_DeleteCurrent(PyThreadState *tstate);
 
-extern void _PyAtExit_Call(PyInterpreterState *interp);
+extern int _PyAtExit_Call(PyInterpreterState *interp);
 
 extern int _Py_IsCoreInitialized(void);
 

Lib/test/test_atexit.py

@@ -79,6 +79,30 @@ def thready():
         # want them to affect the rest of the tests.
         script_helper.assert_python_ok("-c", textwrap.dedent(source))
 
+    @threading_helper.requires_working_threading()
+    def test_thread_created_in_atexit(self):
+        source = """
+        import atexit
+        import threading
+        import time
+
+
+        def run():
+            print(24)
+            time.sleep(1)
+            print(42)
+
+        @atexit.register
+        def start_thread():
+            threading.Thread(target=run).start()
+        """
+
+        return_code, stdout, stderr = script_helper.assert_python_ok("-c", textwrap.dedent(source))
+        self.assertEqual(return_code, 0)
+        end = "\r\n" if os.name == "nt" else "\n"
+        self.assertEqual(stdout, f"24{end}42{end}".encode("utf-8"))
+        self.assertEqual(stderr, b"")
+
 
 @support.cpython_only
 class SubinterpreterTest(unittest.TestCase):

Lib/test/test_capi/test_misc.py

@@ -22,6 +22,7 @@
 from test import support
 from test.support import MISSING_C_DOCSTRINGS
 from test.support import import_helper
+from test.support import script_helper
 from test.support import threading_helper
 from test.support import warnings_helper
 from test.support import requires_limited_api
@@ -1641,6 +1642,37 @@ def subthread():
 
             self.assertEqual(actual, int(interpid))
 
+    @threading_helper.requires_working_threading()
+    def test_pending_call_creates_thread(self):
+        source = """
+        import _testcapi
+        import threading
+        import time
+
+
+        def output():
+            print(24)
+            time.sleep(1)
+            print(42)
+
+
+        def callback():
+            threading.Thread(target=output).start()
+
+
+        def create_pending_call():
+            time.sleep(1)
+            _testcapi.simple_pending_call(callback)
+
+
+        threading.Thread(target=create_pending_call).start()
+        """
+        return_code, stdout, stderr = script_helper.assert_python_ok('-c', textwrap.dedent(source))
+        self.assertEqual(return_code, 0)
+        end = "\r\n" if os.name == "nt" else "\n"
+        self.assertEqual(stdout, f"24{end}42{end}".encode("utf-8"))
+        self.assertEqual(stderr, b"")
+
 
 class SubinterpreterTest(unittest.TestCase):
 

Lib/threading.py

@@ -1557,8 +1557,9 @@ def _shutdown():
     # normally - that won't happen until the interpreter is nearly dead. So
     # mark it done here.
     if _main_thread._os_thread_handle.is_done() and _is_main_interpreter():
-        # _shutdown() was already called
-        return
+        # _shutdown() was already called, but threads might have started
+        # in the meantime.
+        return _thread_shutdown()
 
     global _SHUTTING_DOWN
     _SHUTTING_DOWN = True
@@ -1572,7 +1573,7 @@ def _shutdown():
         _main_thread._os_thread_handle._set_done()
 
     # Wait for all non-daemon threads to exit.
-    _thread_shutdown()
+    return _thread_shutdown()
 
 
 def main_thread():

Misc/NEWS.d/next/Core_and_Builtins/2025-06-26-18-44-34.gh-issue-136003.sln51d.rst

@@ -0,0 +1,3 @@
+Fix :class:`threading.Thread` objects becoming incorrectly daemon when
+created from an :mod:`atexit` callback or a pending call
+(:c:func:`Py_AddPendingCall`).

Modules/_testcapimodule.c

@@ -2546,6 +2546,16 @@ toggle_reftrace_printer(PyObject *ob, PyObject *arg)
     Py_RETURN_NONE;
 }
 
+static PyObject *
+simple_pending_call(PyObject *self, PyObject *callable)
+{
+    if (Py_AddPendingCall(_pending_callback, Py_NewRef(callable)) < 0) {
+        return NULL;
+    }
+
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef TestMethods[] = {
     {"set_errno",               set_errno,                       METH_VARARGS},
     {"test_config",             test_config,                     METH_NOARGS},
@@ -2640,6 +2650,7 @@ static PyMethodDef TestMethods[] = {
     {"test_atexit", test_atexit, METH_NOARGS},
     {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL},
     {"toggle_reftrace_printer", toggle_reftrace_printer, METH_O},
+    {"simple_pending_call", simple_pending_call, METH_O},
     {NULL, NULL} /* sentinel */
 };
 

Modules/_threadmodule.c

@@ -2345,6 +2345,7 @@ thread_shutdown(PyObject *self, PyObject *args)
 {
     PyThread_ident_t ident = PyThread_get_thread_ident_ex();
     thread_module_state *state = get_thread_state(self);
+    int found_thread = 0;
 
     for (;;) {
         ThreadHandle *handle = NULL;
@@ -2353,6 +2354,7 @@ thread_shutdown(PyObject *self, PyObject *args)
         HEAD_LOCK(&_PyRuntime);
         struct llist_node *node;
         llist_for_each_safe(node, &state->shutdown_handles) {
+            found_thread = 1;
             ThreadHandle *cur = llist_data(node, ThreadHandle, shutdown_node);
             if (cur->ident != ident) {
                 ThreadHandle_incref(cur);
@@ -2373,13 +2375,13 @@ thread_shutdown(PyObject *self, PyObject *args)
             PyErr_FormatUnraisable("Exception ignored while joining a thread "
                                    "in _thread._shutdown()");
             ThreadHandle_decref(handle);
-            Py_RETURN_NONE;
+            return PyBool_FromLong(found_thread);
         }
 
         ThreadHandle_decref(handle);
     }
 
-    Py_RETURN_NONE;
+    return PyBool_FromLong(found_thread);
 }
 
 PyDoc_STRVAR(shutdown_doc,

Modules/atexitmodule.c

@@ -99,7 +99,7 @@ _PyAtExit_Fini(PyInterpreterState *interp)
     }
 }
 
-static void
+static int
 atexit_callfuncs(struct atexit_state *state)
 {
     assert(!PyErr_Occurred());
@@ -112,10 +112,13 @@ atexit_callfuncs(struct atexit_state *state)
     {
         PyErr_FormatUnraisable("Exception ignored while "
                                "copying atexit callbacks");
-        return;
+        return 0;
     }
 
+    int called = 0;
+
     for (Py_ssize_t i = 0; i < PyList_GET_SIZE(copy); ++i) {
+        called = 1;
         // We don't have to worry about evil borrowed references, because
         // no other threads can access this list.
         PyObject *tuple = PyList_GET_ITEM(copy, i);
@@ -141,14 +144,15 @@ atexit_callfuncs(struct atexit_state *state)
     atexit_cleanup(state);
 
     assert(!PyErr_Occurred());
+    return called;
 }
 
 
-void
+int
 _PyAtExit_Call(PyInterpreterState *interp)
 {
     struct atexit_state *state = &interp->atexit;
-    atexit_callfuncs(state);
+    return atexit_callfuncs(state);
 }
 
 

Python/ceval_gil.c

@@ -837,10 +837,11 @@ handle_signals(PyThreadState *tstate)
 }
 
 static int
-_make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
+_make_pending_calls(struct _pending_calls *pending, int32_t *p_npending, int *p_called)
 {
     int res = 0;
     int32_t npending = -1;
+    int called = 0;
 
     assert(sizeof(pending->max) <= sizeof(size_t)
             && ((size_t)pending->max) <= Py_ARRAY_LENGTH(pending->calls));
@@ -868,6 +869,8 @@ _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
             break;
         }
 
+        called = 1;
+
         /* having released the lock, perform the callback */
         res = func(arg);
         if ((flags & _Py_PENDING_RAWFREE) && arg != NULL) {
@@ -881,6 +884,7 @@ _make_pending_calls(struct _pending_calls *pending, int32_t *p_npending)
 
 finally:
     *p_npending = npending;
+    *p_called = called;
     return res;
 }
 
@@ -917,7 +921,7 @@ clear_pending_handling_thread(struct _pending_calls *pending)
 }
 
 static int
-make_pending_calls(PyThreadState *tstate)
+make_pending_calls_with_count(PyThreadState *tstate)
 {
     PyInterpreterState *interp = tstate->interp;
     struct _pending_calls *pending = &interp->ceval.pending;
@@ -947,7 +951,8 @@ make_pending_calls(PyThreadState *tstate)
     unsignal_pending_calls(tstate, interp);
 
     int32_t npending;
-    if (_make_pending_calls(pending, &npending) != 0) {
+    int called;
+    if (_make_pending_calls(pending, &npending, &called) != 0) {
         clear_pending_handling_thread(pending);
         /* There might not be more calls to make, but we play it safe. */
         signal_pending_calls(tstate, interp);
@@ -958,8 +963,9 @@ make_pending_calls(PyThreadState *tstate)
         signal_pending_calls(tstate, interp);
     }
 
+    int main_called = 0;
     if (_Py_IsMainThread() && _Py_IsMainInterpreter(interp)) {
-        if (_make_pending_calls(pending_main, &npending) != 0) {
+        if (_make_pending_calls(pending_main, &npending, &main_called) != 0) {
             clear_pending_handling_thread(pending);
             /* There might not be more calls to make, but we play it safe. */
             signal_pending_calls(tstate, interp);
@@ -972,6 +978,16 @@ make_pending_calls(PyThreadState *tstate)
     }
 
     clear_pending_handling_thread(pending);
+    return Py_MAX(called, main_called);
+}
+
+static int
+make_pending_calls(PyThreadState *tstate)
+{
+    if (make_pending_calls_with_count(tstate) < 0) {
+        return -1;
+    }
+
     return 0;
 }
 
@@ -994,7 +1010,7 @@ _Py_unset_eval_breaker_bit_all(PyInterpreterState *interp, uintptr_t bit)
     _Py_FOR_EACH_TSTATE_END(interp);
 }
 
-void
+int
 _Py_FinishPendingCalls(PyThreadState *tstate)
 {
     _Py_AssertHoldsTstate();
@@ -1011,13 +1027,18 @@ _Py_FinishPendingCalls(PyThreadState *tstate)
 #ifndef NDEBUG
     int32_t npending_prev = INT32_MAX;
 #endif
+    int called = 0;
     do {
-        if (make_pending_calls(tstate) < 0) {
+        int res = make_pending_calls_with_count(tstate);
+        if (res < 0) {
             PyObject *exc = _PyErr_GetRaisedException(tstate);
             PyErr_BadInternalCall();
             _PyErr_ChainExceptions1(exc);
             _PyErr_Print(tstate);
         }
+        if (res != 0) {
+            called = 1;
+        }
 
         npending = _Py_atomic_load_int32_relaxed(&pending->npending);
         if (pending_main != NULL) {
@@ -1028,6 +1049,8 @@ _Py_FinishPendingCalls(PyThreadState *tstate)
         npending_prev = npending;
 #endif
     } while (npending > 0);
+
+    return called;
 }
 
 int