[pull] main from python:main

Include/internal/pycore_pymem.h

@@ -55,13 +55,6 @@ struct _Py_mem_interp_free_queue {
     struct llist_node head;  // queue of _mem_work_chunk items
 };
 
-/* Set the memory allocator of the specified domain to the default.
-   Save the old allocator into *old_alloc if it's non-NULL.
-   Return on success, or return -1 if the domain is unknown. */
-extern int _PyMem_SetDefaultAllocator(
-    PyMemAllocatorDomain domain,
-    PyMemAllocatorEx *old_alloc);
-
 /* Special bytes broadcast into debug memory blocks at appropriate times.
    Strings of these are unlikely to be valid addresses, floats, ints or
    7-bit ASCII.
@@ -113,6 +106,13 @@ extern int _PyMem_GetAllocatorName(
    PYMEM_ALLOCATOR_NOT_SET does nothing. */
 extern int _PyMem_SetupAllocators(PyMemAllocatorName allocator);
 
+// Default raw memory allocator that is not affected by PyMem_SetAllocator()
+extern void *_PyMem_DefaultRawMalloc(size_t);
+extern void *_PyMem_DefaultRawCalloc(size_t, size_t);
+extern void *_PyMem_DefaultRawRealloc(void *, size_t);
+extern void _PyMem_DefaultRawFree(void *);
+extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str);
+
 /* Is the debug allocator enabled? */
 extern int _PyMem_DebugEnabled(void);
 
@@ -132,7 +132,6 @@ static inline void _PyObject_XDecRefDelayed(PyObject *obj)
 // Periodically process delayed free requests.
 extern void _PyMem_ProcessDelayed(PyThreadState *tstate);
 
-
 // Periodically process delayed free requests when the world is stopped.
 // Notify of any objects whic should be freeed.
 typedef void (*delayed_dealloc_cb)(PyObject *, void *);

Misc/NEWS.d/next/Build/2025-02-07-21-20-21.gh-issue-129838.fkuiEc.rst

@@ -0,0 +1,2 @@
+Don't redefine ``_Py_NO_SANITIZE_UNDEFINED`` when compiling with a recent
+GCC version and undefined sanitizer enabled.

Modules/_testexternalinspection.c

@@ -133,6 +133,10 @@ return_section_address(
 
         cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize);
     }
+
+    // We should not be here, but if we are there, we should say about this
+    PyErr_SetString(
+        PyExc_RuntimeError, "Cannot find section address.\n");
     return 0;
 }
 
@@ -188,6 +192,8 @@ search_section_in_file(
 
     munmap(map, fs.st_size);
     if (close(fd) != 0) {
+        // This might hide one of the above exceptions, maybe we
+        // should chain them?
         PyErr_SetFromErrno(PyExc_OSError);
     }
     return result;
@@ -217,7 +223,6 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) {
 
     mach_port_t proc_ref = pid_to_task(pid);
     if (proc_ref == 0) {
-        PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID");
         return 0;
     }
 
@@ -260,6 +265,9 @@ search_map_for_section(pid_t pid, const char* secname, const char* substr) {
 
         address += size;
     }
+
+    PyErr_SetString(PyExc_RuntimeError,
+        "mach_vm_region failed to find the section");
     return 0;
 }
 
@@ -306,6 +314,8 @@ find_map_start_address(pid_t pid, char* result_filename, const char* map)
 
     if (!match_found) {
         map_filename[0] = '\0';
+        PyErr_Format(PyExc_RuntimeError,
+            "Cannot find map start address for map: %s", map);
     }
 
     return result_address;
@@ -401,6 +411,8 @@ search_map_for_section(pid_t pid, const char* secname, const char* map)
 static uintptr_t
 search_map_for_section(pid_t pid, const char* secname, const char* map)
 {
+    PyErr_SetString(PyExc_NotImplementedError,
+        "Not supported on this platform");
     return 0;
 }
 #endif
@@ -419,7 +431,8 @@ get_py_runtime(pid_t pid)
 static uintptr_t
 get_async_debug(pid_t pid)
 {
-    uintptr_t result = search_map_for_section(pid, "AsyncioDebug", "_asyncio.cpython");
+    uintptr_t result = search_map_for_section(pid, "AsyncioDebug",
+        "_asyncio.cpython");
     if (result == 0 && !PyErr_Occurred()) {
         PyErr_SetString(PyExc_RuntimeError, "Cannot find AsyncioDebug section");
     }
@@ -482,6 +495,9 @@ read_memory(pid_t pid, uintptr_t remote_address, size_t len, void* dst)
     }
     total_bytes_read = len;
 #else
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        "Memory reading is not supported on this platform");
     return -1;
 #endif
     return total_bytes_read;
@@ -789,6 +805,9 @@ parse_coro_chain(
         pid,
         coro_address + offsets->gen_object.gi_frame_state,
         &gi_frame_state);
+    if (err) {
+        return -1;
+    }
 
     if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) {
         char owner;

Modules/faulthandler.c

@@ -37,14 +37,15 @@
 #define PUTS(fd, str) (void)_Py_write_noraise(fd, str, strlen(str))
 
 
-// clang uses __attribute__((no_sanitize("undefined")))
-// GCC 4.9+ uses __attribute__((no_sanitize_undefined))
-#if defined(__has_feature)  // Clang
+// Clang and GCC 9.0+ use __attribute__((no_sanitize("undefined")))
+#if defined(__has_feature)
 #  if __has_feature(undefined_behavior_sanitizer)
 #    define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize("undefined")))
 #  endif
 #endif
-#if defined(__GNUC__) \
+
+// GCC 4.9+ uses __attribute__((no_sanitize_undefined))
+#if !defined(_Py_NO_SANITIZE_UNDEFINED) && defined(__GNUC__) \
     && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 9))
 #  define _Py_NO_SANITIZE_UNDEFINED __attribute__((no_sanitize_undefined))
 #endif

Objects/dictobject.c

@@ -7163,6 +7163,17 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg)
     return 0;
 }
 
+static void
+clear_inline_values(PyDictValues *values)
+{
+    if (values->valid) {
+        FT_ATOMIC_STORE_UINT8(values->valid, 0);
+        for (Py_ssize_t i = 0; i < values->capacity; i++) {
+            Py_CLEAR(values->values[i]);
+        }
+    }
+}
+
 static void
 set_dict_inline_values(PyObject *obj, PyDictObject *new_dict)
 {
@@ -7173,12 +7184,7 @@ set_dict_inline_values(PyObject *obj, PyDictObject *new_dict)
     Py_XINCREF(new_dict);
     FT_ATOMIC_STORE_PTR(_PyObject_ManagedDictPointer(obj)->dict, new_dict);
 
-    if (values->valid) {
-        FT_ATOMIC_STORE_UINT8(values->valid, 0);
-        for (Py_ssize_t i = 0; i < values->capacity; i++) {
-            Py_CLEAR(values->values[i]);
-        }
-    }
+    clear_inline_values(values);
 }
 
 #ifdef Py_GIL_DISABLED
@@ -7256,8 +7262,8 @@ decref_maybe_delay(PyObject *obj, bool delay)
     }
 }
 
-static int
-set_or_clear_managed_dict(PyObject *obj, PyObject *new_dict, bool clear)
+int
+_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict)
 {
     assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT);
 #ifndef NDEBUG
@@ -7292,8 +7298,7 @@ set_or_clear_managed_dict(PyObject *obj, PyObject *new_dict, bool clear)
 
             // Decref for the dictionary we incref'd in try_set_dict_inline_only_or_other_dict
             // while the object was locked
-            decref_maybe_delay((PyObject *)prev_dict,
-                               !clear && prev_dict != cur_dict);
+            decref_maybe_delay((PyObject *)prev_dict, prev_dict != cur_dict);
             if (err != 0) {
                 return err;
             }
@@ -7303,7 +7308,7 @@ set_or_clear_managed_dict(PyObject *obj, PyObject *new_dict, bool clear)
 
         if (prev_dict != NULL) {
             // decref for the dictionary that we replaced
-            decref_maybe_delay((PyObject *)prev_dict, !clear);
+            decref_maybe_delay((PyObject *)prev_dict, true);
         }
 
         return 0;
@@ -7333,45 +7338,15 @@ set_or_clear_managed_dict(PyObject *obj, PyObject *new_dict, bool clear)
                             (PyDictObject *)Py_XNewRef(new_dict));
 
         Py_END_CRITICAL_SECTION();
-        decref_maybe_delay((PyObject *)dict, !clear);
+        decref_maybe_delay((PyObject *)dict, true);
     }
     assert(_PyObject_InlineValuesConsistencyCheck(obj));
     return err;
 }
 
-int
-_PyObject_SetManagedDict(PyObject *obj, PyObject *new_dict)
-{
-    return set_or_clear_managed_dict(obj, new_dict, false);
-}
-
-void
-PyObject_ClearManagedDict(PyObject *obj)
-{
-    if (set_or_clear_managed_dict(obj, NULL, true) < 0) {
-        /* Must be out of memory */
-        assert(PyErr_Occurred() == PyExc_MemoryError);
-        PyErr_FormatUnraisable("Exception ignored while "
-                               "clearing an object managed dict");
-        /* Clear the dict */
-        PyDictObject *dict = _PyObject_GetManagedDict(obj);
-        Py_BEGIN_CRITICAL_SECTION2(dict, obj);
-        dict = _PyObject_ManagedDictPointer(obj)->dict;
-        PyInterpreterState *interp = _PyInterpreterState_GET();
-        PyDictKeysObject *oldkeys = dict->ma_keys;
-        set_keys(dict, Py_EMPTY_KEYS);
-        dict->ma_values = NULL;
-        dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict));
-        STORE_USED(dict, 0);
-        set_dict_inline_values(obj, NULL);
-        Py_END_CRITICAL_SECTION2();
-    }
-}
-
-int
-_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
+static int
+detach_dict_from_object(PyDictObject *mp, PyObject *obj)
 {
-    ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj);
     assert(_PyObject_ManagedDictPointer(obj)->dict == mp);
     assert(_PyObject_InlineValuesConsistencyCheck(obj));
 
@@ -7401,6 +7376,60 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
     return 0;
 }
 
+
+void
+PyObject_ClearManagedDict(PyObject *obj)
+{
+    // This is called when the object is being freed or cleared
+    // by the GC and therefore known to have no references.
+    if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES) {
+        PyDictObject *dict = _PyObject_GetManagedDict(obj);
+        if (dict == NULL) {
+            // We have no materialized dictionary and inline values
+            // that just need to be cleared.
+            // No dict to clear, we're done
+            clear_inline_values(_PyObject_InlineValues(obj));
+            return;
+        }
+        else if (FT_ATOMIC_LOAD_PTR_RELAXED(dict->ma_values) ==
+                    _PyObject_InlineValues(obj)) {
+            // We have a materialized object which points at the inline
+            // values. We need to materialize the keys. Nothing can modify
+            // this object, but we need to lock the dictionary.
+            int err;
+            Py_BEGIN_CRITICAL_SECTION(dict);
+            err = detach_dict_from_object(dict, obj);
+            Py_END_CRITICAL_SECTION();
+
+            if (err) {
+                /* Must be out of memory */
+                assert(PyErr_Occurred() == PyExc_MemoryError);
+                PyErr_FormatUnraisable("Exception ignored while "
+                                       "clearing an object managed dict");
+                /* Clear the dict */
+                Py_BEGIN_CRITICAL_SECTION(dict);
+                PyInterpreterState *interp = _PyInterpreterState_GET();
+                PyDictKeysObject *oldkeys = dict->ma_keys;
+                set_keys(dict, Py_EMPTY_KEYS);
+                dict->ma_values = NULL;
+                dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(dict));
+                STORE_USED(dict, 0);
+                clear_inline_values(_PyObject_InlineValues(obj));
+                Py_END_CRITICAL_SECTION();
+            }
+        }
+    }
+    Py_CLEAR(_PyObject_ManagedDictPointer(obj)->dict);
+}
+
+int
+_PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj)
+{
+    ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(obj);
+
+    return detach_dict_from_object(mp, obj);
+}
+
 static inline PyObject *
 ensure_managed_dict(PyObject *obj)
 {