Merge branch 'main' into wr-132413

Author: neonene

Doc/library/exceptions.rst

@@ -429,7 +429,9 @@ The following exceptions are the exceptions that are usually raised.
 
    * Creating a new Python thread.
    * :meth:`Joining <threading.Thread.join>` a running daemon thread.
-   * :func:`os.fork`.
+   * :func:`os.fork`,
+   * acquiring a lock such as :class:`threading.Lock`, when it is known that
+     the operation would otherwise deadlock.
 
    See also the :func:`sys.is_finalizing` function.
 
@@ -440,6 +442,11 @@ The following exceptions are the exceptions that are usually raised.
 
       :meth:`threading.Thread.join` can now raise this exception.
 
+   .. versionchanged:: next
+
+      This exception may be raised when acquiring :meth:`threading.Lock`
+      or :meth:`threading.RLock`.
+
 .. exception:: RecursionError
 
    This exception is derived from :exc:`RuntimeError`.  It is raised when the

Include/internal/pycore_lock.h

@@ -51,6 +51,11 @@ typedef enum _PyLockFlags {
 
     // Fail if interrupted by a signal while waiting on the lock.
     _PY_FAIL_IF_INTERRUPTED = 4,
+
+    // Locking & unlocking this lock requires attached thread state.
+    // If locking returns PY_LOCK_FAILURE, a Python exception *may* be raised.
+    // (Intended for use with _PY_LOCK_HANDLE_SIGNALS and _PY_LOCK_DETACH.)
+    _PY_LOCK_PYTHONLOCK = 8,
 } _PyLockFlags;
 
 // Lock a mutex with an optional timeout and additional options. See

Include/pyport.h

@@ -667,25 +667,6 @@ extern "C" {
 #endif
 
 
-// _Py_NO_SANITIZE_UNDEFINED(): Disable Undefined Behavior sanitizer (UBsan)
-// on a function.
-//
-// Clang and GCC 9.0+ use __attribute__((no_sanitize("undefined"))).
-// GCC 4.9+ uses __attribute__((no_sanitize_undefined)).
-#if defined(__has_feature)
-#  if __has_feature(undefined_behavior_sanitizer)
-#    define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
-#  endif
-#endif
-#if !defined(_Py_NO_SANITIZE_UNDEFINED) && defined(__GNUC__) \
-    && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9))
-#  define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
-#endif
-#ifndef _Py_NO_SANITIZE_UNDEFINED
-#  define _Py_NO_SANITIZE_UNDEFINED
-#endif
-
-
 // _Py_NONSTRING: The nonstring variable attribute specifies that an object or
 // member declaration with type array of char, signed char, or unsigned char,
 // or pointer to such a type is intended to store character arrays that do not

Lib/test/test_fileio.py

@@ -591,7 +591,7 @@ def testBytesOpen(self):
         try:
             f.write(b"abc")
             f.close()
-            with open(TESTFN_ASCII, "rb") as f:
+            with self.open(TESTFN_ASCII, "rb") as f:
                 self.assertEqual(f.read(), b"abc")
         finally:
             os.unlink(TESTFN_ASCII)
@@ -608,7 +608,7 @@ def testUtf8BytesOpen(self):
         try:
             f.write(b"abc")
             f.close()
-            with open(TESTFN_UNICODE, "rb") as f:
+            with self.open(TESTFN_UNICODE, "rb") as f:
                 self.assertEqual(f.read(), b"abc")
         finally:
             os.unlink(TESTFN_UNICODE)
@@ -692,13 +692,13 @@ def bug801631():
 
     def testAppend(self):
         try:
-            f = open(TESTFN, 'wb')
+            f = self.FileIO(TESTFN, 'wb')
             f.write(b'spam')
             f.close()
-            f = open(TESTFN, 'ab')
+            f = self.FileIO(TESTFN, 'ab')
             f.write(b'eggs')
             f.close()
-            f = open(TESTFN, 'rb')
+            f = self.FileIO(TESTFN, 'rb')
             d = f.read()
             f.close()
             self.assertEqual(d, b'spameggs')
@@ -734,6 +734,7 @@ def __setattr__(self, name, value):
 class COtherFileTests(OtherFileTests, unittest.TestCase):
     FileIO = _io.FileIO
     modulename = '_io'
+    open = _io.open
 
     @cpython_only
     def testInvalidFd_overflow(self):
@@ -755,6 +756,7 @@ def test_open_code(self):
 class PyOtherFileTests(OtherFileTests, unittest.TestCase):
     FileIO = _pyio.FileIO
     modulename = '_pyio'
+    open = _pyio.open
 
     def test_open_code(self):
         # Check that the default behaviour of open_code matches

Lib/test/test_pyexpat.py

@@ -9,12 +9,11 @@
 from io import BytesIO
 from test import support
 from test.support import os_helper
-
+from test.support import sortdict
+from unittest import mock
 from xml.parsers import expat
 from xml.parsers.expat import errors
 
-from test.support import sortdict
-
 
 class SetAttributeTest(unittest.TestCase):
     def setUp(self):
@@ -436,6 +435,19 @@ def test7(self):
                           "<!--abc-->", "4", "<!--def-->", "5", "</a>"],
                          "buffered text not properly split")
 
+    def test_change_character_data_handler_in_callback(self):
+        # Test that xmlparse_handler_setter() properly handles
+        # the special case "parser.CharacterDataHandler = None".
+        def handler(*args):
+            parser.CharacterDataHandler = None
+
+        handler_wrapper = mock.Mock(wraps=handler)
+        parser = expat.ParserCreate()
+        parser.CharacterDataHandler = handler_wrapper
+        parser.Parse(b"<a>1<b/>2<c></c>3<!--abc-->4<!--def-->5</a> ", True)
+        handler_wrapper.assert_called_once()
+        self.assertIsNone(parser.CharacterDataHandler)
+
 
 # Test handling of exception from callback:
 class HandlerExceptionTest(unittest.TestCase):
@@ -595,7 +607,7 @@ def test_unchanged_size(self):
     def test_disabling_buffer(self):
         xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512
         xml2 = b'b' * 1024
-        xml3 = b'c' * 1024 + b'</a>';
+        xml3 = b'c' * 1024 + b'</a>'
         parser = expat.ParserCreate()
         parser.CharacterDataHandler = self.counting_handler
         parser.buffer_text = 1

Lib/test/test_threading.py

@@ -1247,6 +1247,61 @@ def __del__(self):
         self.assertEqual(err, b"")
         self.assertIn(b"all clear", out)
 
+    @support.subTests('lock_class_name', ['Lock', 'RLock'])
+    def test_acquire_daemon_thread_lock_in_finalization(self, lock_class_name):
+        # gh-123940: Py_Finalize() prevents other threads from running Python
+        # code (and so, releasing locks), so acquiring a locked lock can not
+        # succeed.
+        # We raise an exception rather than hang.
+        code = textwrap.dedent(f"""
+            import threading
+            import time
+
+            thread_started_event = threading.Event()
+
+            lock = threading.{lock_class_name}()
+            def loop():
+                if {lock_class_name!r} == 'RLock':
+                    lock.acquire()
+                with lock:
+                    thread_started_event.set()
+                    while True:
+                        time.sleep(1)
+
+            uncontested_lock = threading.{lock_class_name}()
+
+            class Cycle:
+                def __init__(self):
+                    self.self_ref = self
+                    self.thr = threading.Thread(
+                        target=loop, daemon=True)
+                    self.thr.start()
+                    thread_started_event.wait()
+
+                def __del__(self):
+                    assert self.thr.is_alive()
+
+                    # We *can* acquire an unlocked lock
+                    uncontested_lock.acquire()
+                    if {lock_class_name!r} == 'RLock':
+                        uncontested_lock.acquire()
+
+                    # Acquiring a locked one fails
+                    try:
+                        lock.acquire()
+                    except PythonFinalizationError:
+                        assert self.thr.is_alive()
+                        print('got the correct exception!')
+
+            # Cycle holds a reference to itself, which ensures it is
+            # cleaned up during the GC that runs after daemon threads
+            # have been forced to exit during finalization.
+            Cycle()
+        """)
+        rc, out, err = assert_python_ok("-c", code)
+        self.assertEqual(err, b"")
+        self.assertIn(b"got the correct exception", out)
+
     def test_start_new_thread_failed(self):
         # gh-109746: if Python fails to start newly created thread
         # due to failure of underlying PyThread_start_new_thread() call,

Misc/NEWS.d/next/C_API/2025-06-19-12-47-18.gh-issue-133157.1WA85f.rst

@@ -0,0 +1 @@
+Remove the private, undocumented macro :c:macro:`!_Py_NO_SANITIZE_UNDEFINED`.

Misc/NEWS.d/next/Library/2025-06-27-09-26-04.gh-issue-87135.33z0UW.rst

@@ -0,0 +1,3 @@
+Acquiring a :class:`threading.Lock` or :class:`threading.RLock` at interpreter
+shutdown will raise :exc:`PythonFinalizationError` if Python can determine
+that it would otherwise deadlock.

Modules/_threadmodule.c

@@ -834,9 +834,14 @@ lock_PyThread_acquire_lock(PyObject *op, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyLockStatus r = _PyMutex_LockTimed(&self->lock, timeout,
-                                        _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
+    PyLockStatus r = _PyMutex_LockTimed(
+        &self->lock, timeout,
+        _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
     if (r == PY_LOCK_INTR) {
+        assert(PyErr_Occurred());
+        return NULL;
+    }
+    if (r == PY_LOCK_FAILURE && PyErr_Occurred()) {
         return NULL;
     }
 
@@ -1054,9 +1059,14 @@ rlock_acquire(PyObject *op, PyObject *args, PyObject *kwds)
         return NULL;
     }
 
-    PyLockStatus r = _PyRecursiveMutex_LockTimed(&self->lock, timeout,
-                                                 _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
+    PyLockStatus r = _PyRecursiveMutex_LockTimed(
+        &self->lock, timeout,
+        _PY_LOCK_PYTHONLOCK | _PY_LOCK_HANDLE_SIGNALS | _PY_LOCK_DETACH);
     if (r == PY_LOCK_INTR) {
+        assert(PyErr_Occurred());
+        return NULL;
+    }
+    if (r == PY_LOCK_FAILURE && PyErr_Occurred()) {
         return NULL;
     }
 

Modules/pyexpat.c

@@ -98,7 +98,11 @@ typedef struct {
 
 #define CHARACTER_DATA_BUFFER_SIZE 8192
 
-typedef const void *xmlhandler;
+// A generic function type for storage.
+// To avoid undefined behaviors, a handler must be cast to the correct
+// function type before it's called; see SETTER_WRAPPER below.
+typedef void (*xmlhandler)(void);
+
 typedef void (*xmlhandlersetter)(XML_Parser self, xmlhandler handler);
 
 struct HandlerInfo {
@@ -110,9 +114,7 @@ struct HandlerInfo {
 
 static struct HandlerInfo handler_info[64];
 
-// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED, rather than using the exact
-// handler API for each handler.
-static inline void _Py_NO_SANITIZE_UNDEFINED
+static inline void
 CALL_XML_HANDLER_SETTER(const struct HandlerInfo *handler_info,
                         XML_Parser xml_parser, xmlhandler xml_handler)
 {
@@ -1365,7 +1367,7 @@ xmlparse_handler_setter(PyObject *op, PyObject *v, void *closure)
            elaborate system of handlers and state could remove the
            C handler more effectively. */
         if (handlernum == CharacterData && self->in_callback) {
-            c_handler = noop_character_data_handler;
+            c_handler = (xmlhandler)noop_character_data_handler;
         }
         v = NULL;
     }
@@ -2222,13 +2224,84 @@ clear_handlers(xmlparseobject *self, int initial)
     }
 }
 
+/* To avoid undefined behaviors, a function must be *called* via a function
+ * pointer of the correct type.
+ * So, for each `XML_Set*` function, we define a wrapper that calls `XML_Set*`
+ * with its argument cast to the appropriate type.
+ */
+
+typedef void (*parser_only)(void *);
+typedef int (*not_standalone)(void *);
+typedef void (*parser_and_data)(void *, const XML_Char *);
+typedef void (*parser_and_data_and_int)(void *, const XML_Char *, int);
+typedef void (*parser_and_data_and_data)(
+    void *, const XML_Char *, const XML_Char *);
+typedef void (*start_element)(void *, const XML_Char *, const XML_Char **);
+typedef void (*element_decl)(void *, const XML_Char *, XML_Content *);
+typedef void (*xml_decl)(
+    void *, const XML_Char *, const XML_Char *, int);
+typedef void (*start_doctype_decl)(
+    void *, const XML_Char *, const XML_Char *, const XML_Char *, int);
+typedef void (*notation_decl)(
+    void *,
+    const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *);
+typedef void (*attlist_decl)(
+    void *,
+    const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *,
+    int);
+typedef void (*unparsed_entity_decl)(
+    void *,
+    const XML_Char *, const XML_Char *,
+    const XML_Char *, const XML_Char *, const XML_Char *);
+typedef void (*entity_decl)(
+    void *,
+    const XML_Char *, int,
+    const XML_Char *, int,
+    const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *);
+typedef int (*external_entity_ref)(
+    XML_Parser,
+    const XML_Char *, const XML_Char *, const XML_Char *, const XML_Char *);
+
+#define SETTER_WRAPPER(NAME, TYPE)                                      \
+    static inline void                                                  \
+    pyexpat_Set ## NAME (XML_Parser parser, xmlhandler handler)         \
+    {                                                                   \
+        (void)XML_Set ## NAME (parser, (TYPE)handler);                  \
+    }
+
+SETTER_WRAPPER(StartElementHandler, start_element)
+SETTER_WRAPPER(EndElementHandler, parser_and_data)
+SETTER_WRAPPER(ProcessingInstructionHandler, parser_and_data_and_data)
+SETTER_WRAPPER(CharacterDataHandler, parser_and_data_and_int)
+SETTER_WRAPPER(UnparsedEntityDeclHandler, unparsed_entity_decl)
+SETTER_WRAPPER(NotationDeclHandler, notation_decl)
+SETTER_WRAPPER(StartNamespaceDeclHandler, parser_and_data_and_data)
+SETTER_WRAPPER(EndNamespaceDeclHandler, parser_and_data)
+SETTER_WRAPPER(CommentHandler, parser_and_data)
+SETTER_WRAPPER(StartCdataSectionHandler, parser_only)
+SETTER_WRAPPER(EndCdataSectionHandler, parser_only)
+SETTER_WRAPPER(DefaultHandler, parser_and_data_and_int)
+SETTER_WRAPPER(DefaultHandlerExpand, parser_and_data_and_int)
+SETTER_WRAPPER(NotStandaloneHandler, not_standalone)
+SETTER_WRAPPER(ExternalEntityRefHandler, external_entity_ref)
+SETTER_WRAPPER(StartDoctypeDeclHandler, start_doctype_decl)
+SETTER_WRAPPER(EndDoctypeDeclHandler, parser_only)
+SETTER_WRAPPER(EntityDeclHandler, entity_decl)
+SETTER_WRAPPER(XmlDeclHandler, xml_decl)
+SETTER_WRAPPER(ElementDeclHandler, element_decl)
+SETTER_WRAPPER(AttlistDeclHandler, attlist_decl)
+#if XML_COMBINED_VERSION >= 19504
+SETTER_WRAPPER(SkippedEntityHandler, parser_and_data_and_int)
+#endif
+#undef SETTER_WRAPPER
+
 static struct HandlerInfo handler_info[] = {
 
     // The cast to `xmlhandlersetter` is needed as the signature of XML
     // handler functions is not compatible with `xmlhandlersetter` since
     // their second parameter is narrower than a `const void *`.
 #define HANDLER_INFO(name) \
-    {#name, (xmlhandlersetter)XML_Set##name, my_##name},
+    {#name, (xmlhandlersetter)pyexpat_Set##name, (xmlhandler)my_##name},
 
     HANDLER_INFO(StartElementHandler)
     HANDLER_INFO(EndElementHandler)