gh-87135: Raise PythonFinalizationError when joining a blocked daemon thread

If `Py_IsFinalizing()` is true, non-daemon threads (other than the current one)
are done, and daemon threads are prevented from running, so they
cannot finalize themselves and become done. Joining them without timeout
would block forever.

Raise PythonFinalizationError instead of hanging.

See gh-123940 for a use case: calling `join()` from `__del__`. This is
ill-advised, but an exception should at least make it easier to diagnose.

Author: encukou

Doc/c-api/init.rst

@@ -1112,7 +1112,7 @@ Cautions regarding runtime finalization
 In the late stage of :term:`interpreter shutdown`, after attempting to wait for
 non-daemon threads to exit (though this can be interrupted by
 :class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime
-is marked as *finalizing*: :c:func:`_Py_IsFinalizing` and
+is marked as *finalizing*: :c:func:`Py_IsFinalizing` and
 :func:`sys.is_finalizing` return true.  At this point, only the *finalization
 thread* that initiated finalization (typically the main thread) is allowed to
 acquire the :term:`GIL`.

Doc/library/exceptions.rst

@@ -426,13 +426,18 @@ The following exceptions are the exceptions that are usually raised.
    :exc:`PythonFinalizationError` during the Python finalization:
 
    * Creating a new Python thread.
+   * :meth:`Joining <threading.Thread.join>` a running daemon thread
+     without a timeout.
    * :func:`os.fork`.
 
    See also the :func:`sys.is_finalizing` function.
 
    .. versionadded:: 3.13
       Previously, a plain :exc:`RuntimeError` was raised.
 
+   .. versionchanged:: next
+
+      :meth:`threading.Thread.join` can now raise this exception.
 
 .. exception:: RecursionError
 

Doc/library/threading.rst

@@ -435,6 +435,17 @@ since it is impossible to detect the termination of alien threads.
       an error to :meth:`~Thread.join` a thread before it has been started
       and attempts to do so raise the same exception.
 
+      In late stages of :term:`Python finalization <interpreter shutdown>`,
+      if *timeout* is ``None`` and an attempt is made to join a running
+      daemonic thread, :meth:`!join` raises a :exc:`PythonFinalizationError`.
+      (Such a join would block forever: at this point, threads other than the
+      current one are prevented from running Python code and so they cannot
+      finalize themselves.)
+
+      .. versionchanged:: next
+
+         May raise :exc:`PythonFinalizationError`.
+
    .. attribute:: name
 
       A string used for identification purposes only. It has no semantics.

Lib/test/test_threading.py

@@ -1171,6 +1171,99 @@ def __del__(self):
         self.assertEqual(out.strip(), b"OK")
         self.assertIn(b"can't create new thread at interpreter shutdown", err)
 
+    def test_join_daemon_thread_in_finalization(self):
+        # gh-123940: Py_Finalize() prevents other threads from running Python
+        # code, so join() can not succeed unless the thread is already done.
+        # (Non-Python threads, that is `threading._DummyThread`, can't be
+        # joined at all.)
+        # We raise an exception rather than hang.
+        code = textwrap.dedent("""
+            import threading
+
+
+            def loop():
+                while True:
+                    pass
+
+
+            class Cycle:
+                def __init__(self):
+                    self.self_ref = self
+                    self.thr = threading.Thread(target=loop, daemon=True)
+                    self.thr.start()
+
+                def __del__(self):
+                    try:
+                        self.thr.join()
+                    except PythonFinalizationError:
+                        print('got the correct exception!')
+
+            # Cycle holds a reference to itself, which ensures it is cleaned
+            # up during the GC that runs after daemon threads have been
+            # forced to exit during finalization.
+            Cycle()
+        """)
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertEqual(err, b"")
+        self.assertIn(b"got the correct exception", out)
+
+    def test_join_finished_daemon_thread_in_finalization(self):
+        # (see previous test)
+        # If the thread is already finished, join() succeeds.
+        code = textwrap.dedent("""
+            import threading
+            done = threading.Event()
+
+            def loop():
+                done.set()
+
+
+            class Cycle:
+                def __init__(self):
+                    self.self_ref = self
+                    self.thr = threading.Thread(target=loop, daemon=True)
+                    self.thr.start()
+                    done.wait()
+
+                def __del__(self):
+                    self.thr.join()
+                    print('all clear!')
+
+            Cycle()
+        """)
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertEqual(err, b"")
+        self.assertIn(b"all clear", out)
+
+    def test_timed_join_daemon_thread_in_finalization(self):
+        # (see previous test)
+        # When called with timeout, no error is raised.
+        code = textwrap.dedent("""
+            import threading
+            done = threading.Event()
+
+            def loop():
+                done.set()
+                while True:
+                    pass
+
+            class Cycle:
+                def __init__(self):
+                    self.self_ref = self
+                    self.thr = threading.Thread(target=loop, daemon=True)
+                    self.thr.start()
+                    done.wait()
+
+                def __del__(self):
+                    self.thr.join(timeout=0.01)
+                    print('alive:', self.thr.is_alive())
+
+            Cycle()
+        """)
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertEqual(err, b"")
+        self.assertIn(b"alive: True", out)
+
     def test_start_new_thread_failed(self):
         # gh-109746: if Python fails to start newly created thread
         # due to failure of underlying PyThread_start_new_thread() call,

Modules/_threadmodule.c

@@ -510,32 +510,52 @@ ThreadHandle_join(ThreadHandle *self, PyTime_t timeout_ns)
     // To work around this, we set `thread_is_exiting` immediately before
     // `thread_run` returns.  We can be sure that we are not attempting to join
     // ourselves if the handle's thread is about to exit.
-    if (!_PyEvent_IsSet(&self->thread_is_exiting) &&
-        ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) {
-        // PyThread_join_thread() would deadlock or error out.
-        PyErr_SetString(ThreadError, "Cannot join current thread");
-        return -1;
-    }
-
-    // Wait until the deadline for the thread to exit.
-    PyTime_t deadline = timeout_ns != -1 ? _PyDeadline_Init(timeout_ns) : 0;
-    int detach = 1;
-    while (!PyEvent_WaitTimed(&self->thread_is_exiting, timeout_ns, detach)) {
-        if (deadline) {
-            // _PyDeadline_Get will return a negative value if the deadline has
-            // been exceeded.
-            timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0);
+    PyEvent *is_exiting = &self->thread_is_exiting;
+    if (!_PyEvent_IsSet(is_exiting)) {
+        if (ThreadHandle_ident(self) == PyThread_get_thread_ident_ex()) {
+            // PyThread_join_thread() would deadlock or error out.
+            PyErr_SetString(ThreadError, "Cannot join current thread");
+            return -1;
         }
 
-        if (timeout_ns) {
-            // Interrupted
-            if (Py_MakePendingCalls() < 0) {
+        PyTime_t deadline = 0;
+
+        if (timeout_ns == -1) {
+            if (Py_IsFinalizing()) {
+                // gh-123940: On finalization, other threads are prevented from
+                // running Python code. They cannot finalize themselves,
+                // so join() would hang forever.
+                // We raise instead.
+                // (We only do this if no timeout is given: otherwise
+                //  we assume the caller can handle a hung thread.)
+                PyErr_SetString(PyExc_PythonFinalizationError,
+                                "cannot join thread at interpreter shutdown");
                 return -1;
             }
         }
         else {
-            // Timed out
-            return 0;
+            deadline = _PyDeadline_Init(timeout_ns);
+        }
+
+        // Wait until the deadline for the thread to exit.
+        int detach = 1;
+        while (!PyEvent_WaitTimed(is_exiting, timeout_ns, detach)) {
+            if (deadline) {
+                // _PyDeadline_Get will return a negative value if
+                // the deadline has been exceeded.
+                timeout_ns = Py_MAX(_PyDeadline_Get(deadline), 0);
+            }
+
+            if (timeout_ns) {
+                // Interrupted
+                if (Py_MakePendingCalls() < 0) {
+                    return -1;
+                }
+            }
+            else {
+                // Timed out
+                return 0;
+            }
         }
     }