From 9b5e80000ee179eb028841709f10dac9af7c36e7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 27 May 2025 09:42:24 -0600 Subject: [PATCH 1/7] gh-132775: Always Set __builtins__ In _PyFunction_FromXIData() (gh-134758) This is a small follow-up to gh-133481. There's a corner case in the behavior of PyImport_ImportModuleAttrString(), where it expects __builtins__ to be set if __globals__ is set. --- Python/crossinterp_data_lookup.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 88eb41da89ee40..b16f38b847fc66 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -701,6 +701,14 @@ _PyFunction_FromXIData(_PyXIData_t *xidata) Py_DECREF(code); return NULL; } + PyThreadState *tstate = _PyThreadState_GET(); + if (PyDict_SetItem(globals, &_Py_ID(__builtins__), + tstate->interp->builtins) < 0) + { + Py_DECREF(code); + Py_DECREF(globals); + return NULL; + } PyObject *func = PyFunction_New(code, globals); Py_DECREF(code); Py_DECREF(globals); From 3f9eb55e090a8de80503e565f508f341c5f4c8da Mon Sep 17 00:00:00 2001 From: Emma Smith Date: Tue, 27 May 2025 09:02:17 -0700 Subject: [PATCH 2/7] gh-134262: increase retries in `Tools/build/generate_sbom.py` (#134558) --- Tools/build/generate_sbom.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 5845f2d85c7fdb..ecb7b54f6d8a13 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,6 +4,7 @@ import hashlib import json import os +import random import re import subprocess import sys @@ -164,16 +165,18 @@ def get_externals() -> list[str]: def download_with_retries(download_location: str, - max_retries: int = 5, - base_delay: float = 2.0) -> typing.Any: + max_retries: int = 7, + base_delay: float = 2.25, + max_jitter: float = 1.0) -> typing.Any: """Download a file with exponential backoff retry.""" for attempt in range(max_retries): try: resp = urllib.request.urlopen(download_location) except urllib.error.URLError as ex: if attempt == max_retries: - raise ex - time.sleep(base_delay**attempt) + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(base_delay**attempt + random.uniform(0, max_jitter)) else: return resp From a380d578737be1cd51e1d1be2b83bbc0b0619e7e Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 27 May 2025 22:28:27 +0530 Subject: [PATCH 3/7] gh-134043: use stackrefs in vectorcalling methods (#134044) Adds `_PyObject_GetMethodStackRef` which uses stackrefs and takes advantage of deferred reference counting in free-threading while calling method objects in vectorcall. --- Include/internal/pycore_object.h | 3 + Objects/call.c | 35 +++++--- Objects/object.c | 110 +++++++++++++++++++++++++ Tools/ftscalingbench/ftscalingbench.py | 13 +++ 4 files changed, 148 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b7e162c8abcabf..3aaee7d008155a 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -897,6 +897,9 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, + PyObject *name, _PyStackRef *method); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // diff --git a/Objects/call.c b/Objects/call.c index b1610dababd466..c9a18bcc3da60b 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -834,12 +834,15 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, assert(PyVectorcall_NARGS(nargsf) >= 1); PyThreadState *tstate = _PyThreadState_GET(); - PyObject *callable = NULL; + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); /* Use args[0] as "self" argument */ - int unbound = _PyObject_GetMethod(args[0], name, &callable); - if (callable == NULL) { + int unbound = _PyObject_GetMethodStackRef(tstate, args[0], name, &method.ref); + if (PyStackRef_IsNull(method.ref)) { + _PyThreadState_PopCStackRef(tstate, &method); return NULL; } + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); if (unbound) { /* We must remove PY_VECTORCALL_ARGUMENTS_OFFSET since @@ -855,7 +858,7 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable); PyObject *result = _PyObject_VectorcallTstate(tstate, callable, args, nargsf, kwnames); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); return result; } @@ -868,11 +871,14 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) return null_error(tstate); } - PyObject *callable = NULL; - int is_method = _PyObject_GetMethod(obj, name, &callable); - if (callable == NULL) { + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); + int is_method = _PyObject_GetMethodStackRef(tstate, obj, name, &method.ref); + if (PyStackRef_IsNull(method.ref)) { + _PyThreadState_PopCStackRef(tstate, &method); return NULL; } + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); obj = is_method ? obj : NULL; va_list vargs; @@ -880,7 +886,7 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) PyObject *result = object_vacall(tstate, obj, callable, vargs); va_end(vargs); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); return result; } @@ -897,12 +903,15 @@ _PyObject_CallMethodIdObjArgs(PyObject *obj, _Py_Identifier *name, ...) if (!oname) { return NULL; } - - PyObject *callable = NULL; - int is_method = _PyObject_GetMethod(obj, oname, &callable); - if (callable == NULL) { + _PyCStackRef method; + _PyThreadState_PushCStackRef(tstate, &method); + int is_method = _PyObject_GetMethodStackRef(tstate, obj, oname, &method.ref); + if (PyStackRef_IsNull(method.ref)) { + _PyThreadState_PopCStackRef(tstate, &method); return NULL; } + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); + obj = is_method ? obj : NULL; va_list vargs; @@ -910,7 +919,7 @@ _PyObject_CallMethodIdObjArgs(PyObject *obj, _Py_Identifier *name, ...) PyObject *result = object_vacall(tstate, obj, callable, vargs); va_end(vargs); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); return result; } diff --git a/Objects/object.c b/Objects/object.c index af1aa217f75462..68c8bfeae33e33 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1664,6 +1664,116 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } +int +_PyObject_GetMethodStackRef(PyThreadState *ts, PyObject *obj, + PyObject *name, _PyStackRef *method) +{ + int meth_found = 0; + + assert(PyStackRef_IsNull(*method)); + + PyTypeObject *tp = Py_TYPE(obj); + if (!_PyType_IsReady(tp)) { + if (PyType_Ready(tp) < 0) { + return 0; + } + } + + if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_CheckExact(name)) { + PyObject *res = PyObject_GetAttr(obj, name); + if (res != NULL) { + *method = PyStackRef_FromPyObjectSteal(res); + } + return 0; + } + + _PyType_LookupStackRefAndVersion(tp, name, method); + PyObject *descr = PyStackRef_AsPyObjectBorrow(*method); + descrgetfunc f = NULL; + if (descr != NULL) { + if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + meth_found = 1; + } + else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + PyObject *value = f(descr, obj, (PyObject *)Py_TYPE(obj)); + PyStackRef_CLEAR(*method); + if (value != NULL) { + *method = PyStackRef_FromPyObjectSteal(value); + } + return 0; + } + } + } + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + PyStackRef_CLEAR(*method); + *method = PyStackRef_FromPyObjectSteal(attr); + return 0; + } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } + else { + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr != NULL) { + dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + } + else { + dict = NULL; + } + } + if (dict != NULL) { + // TODO: use _Py_dict_lookup_threadsafe_stackref + Py_INCREF(dict); + PyObject *value; + if (PyDict_GetItemRef(dict, name, &value) != 0) { + // found or error + Py_DECREF(dict); + PyStackRef_CLEAR(*method); + if (value != NULL) { + *method = PyStackRef_FromPyObjectSteal(value); + } + return 0; + } + // not found + Py_DECREF(dict); + } + + if (meth_found) { + assert(!PyStackRef_IsNull(*method)); + return 1; + } + + if (f != NULL) { + PyObject *value = f(descr, obj, (PyObject *)Py_TYPE(obj)); + PyStackRef_CLEAR(*method); + if (value) { + *method = PyStackRef_FromPyObjectSteal(value); + } + return 0; + } + + if (descr != NULL) { + assert(!PyStackRef_IsNull(*method)); + return 0; + } + + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + + _PyObject_SetAttributeErrorContext(obj, name); + assert(PyStackRef_IsNull(*method)); + return 0; +} + + /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */ PyObject * diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 926bc66b944c6f..1a59e25189d5dd 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,7 @@ import sys import threading import time +from operator import methodcaller # The iterations in individual benchmarks are scaled by this factor. WORK_SCALE = 100 @@ -188,6 +189,18 @@ def thread_local_read(): _ = tmp.x _ = tmp.x +class MyClass: + __slots__ = () + + def func(self): + pass + +@register_benchmark +def method_caller(): + mc = methodcaller("func") + obj = MyClass() + for i in range(1000 * WORK_SCALE): + mc(obj) def bench_one_thread(func): t0 = time.perf_counter_ns() From ac539e7e0d1622c61988f30c91acfd4c06521ced Mon Sep 17 00:00:00 2001 From: Kumar Aditya Date: Tue, 27 May 2025 22:42:08 +0530 Subject: [PATCH 4/7] gh-132917: fix data race on `last_mem` in free-threading gc (#134692) --- Python/gc_free_threading.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index d2ea5b5e06ba43..5aaa68c5b51f95 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -2062,7 +2062,7 @@ gc_should_collect_mem_usage(GCState *gcstate) // 70,000 new container objects. return true; } - Py_ssize_t last_mem = gcstate->last_mem; + Py_ssize_t last_mem = _Py_atomic_load_ssize_relaxed(&gcstate->last_mem); Py_ssize_t mem_threshold = Py_MAX(last_mem / 10, 128); if ((mem - last_mem) > mem_threshold) { // The process memory usage has increased too much, do a collection. @@ -2245,7 +2245,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Store the current memory usage, can be smaller now if breaking cycles // freed some memory. - state->gcstate->last_mem = get_process_mem_usage(); + Py_ssize_t last_mem = get_process_mem_usage(); + _Py_atomic_store_ssize_relaxed(&state->gcstate->last_mem, last_mem); // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); From 79d81f7cba236e2cbdfa95d0de0a75ba9716ed32 Mon Sep 17 00:00:00 2001 From: Noam Cohen Date: Tue, 27 May 2025 21:30:17 +0300 Subject: [PATCH 5/7] gh-131798: Optimize `_ITER_CHECK_TUPLE` (GH-134803) --- .../2025-05-27-20-21-34.gh-issue-131798.b32zkl.rst | 1 + Python/optimizer_bytecodes.c | 7 +++++++ Python/optimizer_cases.c.h | 6 ++++++ 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-20-21-34.gh-issue-131798.b32zkl.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-20-21-34.gh-issue-131798.b32zkl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-20-21-34.gh-issue-131798.b32zkl.rst new file mode 100644 index 00000000000000..ed4b31bd7bedce --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-20-21-34.gh-issue-131798.b32zkl.rst @@ -0,0 +1 @@ +Allow the JIT to remove unnecessary ``_ITER_CHECK_TUPLE`` ops. diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 34250fd4385d34..e1209209660f92 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -914,6 +914,13 @@ dummy_func(void) { } } + op(_ITER_CHECK_TUPLE, (iter, null_or_index -- iter, null_or_index)) { + if (sym_matches_type(iter, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); + } + op(_ITER_NEXT_RANGE, (iter, null_or_index -- iter, null_or_index, next)) { next = sym_new_type(ctx, &PyLong_Type); } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index bbd45067103679..db86edcc7859b5 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1615,6 +1615,12 @@ } case _ITER_CHECK_TUPLE: { + JitOptSymbol *iter; + iter = stack_pointer[-2]; + if (sym_matches_type(iter, &PyTuple_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + } + sym_set_type(iter, &PyTuple_Type); break; } From 967f361993c9c97eb3ff3076a409b78ea32938df Mon Sep 17 00:00:00 2001 From: Rishabh Singh <67859818+rishabh11336@users.noreply.github.com> Date: Wed, 28 May 2025 00:18:04 +0530 Subject: [PATCH 6/7] gh-134789: Document del s[i] operation for mutable sequences (#134804) [main] Update stdtypes.rst - Added explicit mention of `del s[i]` (item deletion by index) to the Mutable Sequence Types section. - Clarified that this operation removes the item at the specified index from the sequence. - Addresses issue #134789. --- Doc/library/stdtypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 31d71031bca12c..f0b4b09ff10dce 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1214,6 +1214,8 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s[i] = x`` | item *i* of *s* is replaced by | | | | *x* | | +------------------------------+--------------------------------+---------------------+ +| ``del s[i]`` | removes item *i* of *s* | | ++------------------------------+--------------------------------+---------------------+ | ``s[i:j] = t`` | slice of *s* from *i* to *j* | | | | is replaced by the contents of | | | | the iterable *t* | | From a4d37f88b66bc9a66b2ab277aa66a2a6b20821fa Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Tue, 27 May 2025 16:21:16 -0400 Subject: [PATCH 7/7] gh-134679: Fix assertion failure in QSBR (gh-134811) This is the same underlying bug as gh-130519. The destructor may call arbitrary code, changing the `tstate->qsbr pointer` and invalidating the old `struct _qsbr_thread_state`. --- ...-05-27-18-59-54.gh-issue-134679.FWPBu6.rst | 2 ++ Objects/obmalloc.c | 22 +++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-18-59-54.gh-issue-134679.FWPBu6.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-18-59-54.gh-issue-134679.FWPBu6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-18-59-54.gh-issue-134679.FWPBu6.rst new file mode 100644 index 00000000000000..22f1282fea13e9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-27-18-59-54.gh-issue-134679.FWPBu6.rst @@ -0,0 +1,2 @@ +Fix crash in the :term:`free threading` build's QSBR code that could occur +when changing an object's ``__dict__`` attribute. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b209808da902da..d3931aab623b70 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1238,7 +1238,7 @@ work_queue_first(struct llist_node *head) } static void -process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, +process_queue(struct llist_node *head, _PyThreadStateImpl *tstate, bool keep_empty, delayed_dealloc_cb cb, void *state) { while (!llist_empty(head)) { @@ -1246,7 +1246,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, if (buf->rd_idx < buf->wr_idx) { struct _mem_work_item *item = &buf->array[buf->rd_idx]; - if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + if (!_Py_qsbr_poll(tstate->qsbr, item->qsbr_goal)) { return; } @@ -1270,11 +1270,11 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, static void process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { assert(PyMutex_IsLocked(&queue->mutex)); - process_queue(&queue->head, qsbr, false, cb, state); + process_queue(&queue->head, tstate, false, cb, state); int more_work = !llist_empty(&queue->head); _Py_atomic_store_int_relaxed(&queue->has_work, more_work); @@ -1282,7 +1282,7 @@ process_interp_queue(struct _Py_mem_interp_free_queue *queue, static void maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { @@ -1291,7 +1291,7 @@ maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, // Try to acquire the lock, but don't block if it's already held. if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { - process_interp_queue(queue, qsbr, cb, state); + process_interp_queue(queue, tstate, cb, state); PyMutex_Unlock(&queue->mutex); } } @@ -1303,10 +1303,10 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, NULL, NULL); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, NULL, NULL); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, NULL, NULL); } void @@ -1316,10 +1316,10 @@ _PyMem_ProcessDelayedNoDealloc(PyThreadState *tstate, delayed_dealloc_cb cb, voi _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, cb, state); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, cb, state); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, cb, state); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, cb, state); } void @@ -1348,7 +1348,7 @@ _PyMem_AbandonDelayed(PyThreadState *tstate) // Process the merged queue now (see gh-130794). _PyThreadStateImpl *this_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); - process_interp_queue(&interp->mem_free_queue, this_tstate->qsbr, NULL, NULL); + process_interp_queue(&interp->mem_free_queue, this_tstate, NULL, NULL); PyMutex_Unlock(&interp->mem_free_queue.mutex);