diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index badc97808c6132..9cdaa950e3479a 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -939,7 +939,7 @@ struct _is { struct _PyExecutorObject *executor_deletion_list_head; struct _PyExecutorObject *cold_executor; int executor_deletion_list_remaining_capacity; - size_t trace_run_counter; + size_t executor_creation_counter; _rare_events rare_events; PyDict_WatchCallback builtins_dict_watcher; diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 685c39dcd65fb9..8ed5436eb6838c 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -90,8 +90,9 @@ PyAPI_FUNC(void) _Py_Executors_InvalidateCold(PyInterpreterState *interp); #endif // Used as the threshold to trigger executor invalidation when -// trace_run_counter is greater than this value. -#define JIT_CLEANUP_THRESHOLD 100000 +// executor_creation_counter is greater than this value. +// This value is arbitrary and was not optimized. +#define JIT_CLEANUP_THRESHOLD 1000 #define TRACE_STACK_SIZE 5 diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 503cddc73fc464..cab458f84028e2 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -89,7 +89,7 @@ _Py_ThreadCanHandleSignals(PyInterpreterState *interp) /* Variable and static inline functions for in-line access to current thread and interpreter state */ -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) extern _Py_thread_local PyThreadState *_Py_tss_tstate; extern _Py_thread_local PyInterpreterState *_Py_tss_interp; #endif @@ -115,7 +115,7 @@ PyAPI_FUNC(PyThreadState *) _PyThreadState_GetCurrent(void); static inline PyThreadState* _PyThreadState_GET(void) { -#if defined(HAVE_THREAD_LOCAL) && !defined(Py_BUILD_CORE_MODULE) +#if !defined(Py_BUILD_CORE_MODULE) return _Py_tss_tstate; #else return _PyThreadState_GetCurrent(); diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index a4f125e073d3d1..8357cce9d899fb 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -103,6 +103,8 @@ extern int _Py_WriteIndent(int, PyObject *); PyAPI_FUNC(void) _Py_InitDumpStack(void); PyAPI_FUNC(void) _Py_DumpStack(int fd); +extern void _Py_DumpTraceback_Init(void); + #ifdef __cplusplus } #endif diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 9ae72743919a32..423178e87a8684 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -617,6 +617,8 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] return bytes(b) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f121f27174875e..4e94f62d35eba2 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -2637,6 +2637,30 @@ def f2(): f2() + def test_next_instr_for_exception_handler_set(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + for i in range(TIER2_THRESHOLD + 3): + try: + undefined_variable(i) + except Exception: + pass + + f() + + def test_next_instr_for_exception_handler_set_lasts_instr(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + a_list = [] + for _ in range(TIER2_THRESHOLD + 3): + try: + a_list[""] = 0 + except Exception: + pass + + f() + + def global_identity(x): return x diff --git a/Lib/test/test_io/test_general.py b/Lib/test/test_io/test_general.py index ac9c5a425d7ea2..a1cdd6876c2892 100644 --- a/Lib/test/test_io/test_general.py +++ b/Lib/test/test_io/test_general.py @@ -592,6 +592,22 @@ def test_RawIOBase_read(self): self.assertEqual(rawio.read(2), None) self.assertEqual(rawio.read(2), b"") + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.io.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + def test_types_have_dict(self): test = ( self.IOBase(), diff --git a/Lib/test/test_os/test_posix.py b/Lib/test/test_os/test_posix.py index de24719a1ca455..37da293a441e46 100644 --- a/Lib/test/test_os/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -1427,6 +1427,14 @@ def test_sched_param(self): self.assertNotEqual(newparam, param) self.assertEqual(newparam.sched_priority, 0) + @requires_sched + def test_bug_140634(self): + sched_priority = float('inf') # any new reference + param = posix.sched_param(sched_priority) + param.__reduce__() + del sched_priority, param # should not crash + support.gc_collect() # just to be sure + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst new file mode 100644 index 00000000000000..1c18cbc9ad0588 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-10-16-21-47-00.gh-issue-140104.A8SQIm.rst @@ -0,0 +1,2 @@ +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. diff --git a/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst b/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst new file mode 100644 index 00000000000000..cc33217c9f563e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-25-21-04-00.gh-issue-140607.oOZGxS.rst @@ -0,0 +1,2 @@ +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned by +:meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). diff --git a/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst b/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst new file mode 100644 index 00000000000000..b1ba9b26ad5431 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-27-13-49-31.gh-issue-140634.ULng9G.rst @@ -0,0 +1 @@ +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. diff --git a/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst b/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst new file mode 100644 index 00000000000000..299e9f04df7c39 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-10-27-16-01-41.gh-issue-125434.qy0uRA.rst @@ -0,0 +1,2 @@ +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index acadbcc4d59c38..e304fc8bee2bea 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -939,14 +939,21 @@ _io__RawIOBase_read_impl(PyObject *self, Py_ssize_t n) return res; } - n = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_ssize_t bytes_filled = PyNumber_AsSsize_t(res, PyExc_ValueError); Py_DECREF(res); - if (n == -1 && PyErr_Occurred()) { + if (bytes_filled == -1 && PyErr_Occurred()) { Py_DECREF(b); return NULL; } + if (bytes_filled < 0 || bytes_filled > n) { + Py_DECREF(b); + PyErr_Format(PyExc_ValueError, + "readinto returned %zd outside buffer size %zd", + bytes_filled, n); + return NULL; + } - res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n); + res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), bytes_filled); Py_DECREF(b); return res; } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a30712f75d5d06..50464b01efba31 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -8724,7 +8724,7 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) static PyObject * os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { - return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); + return Py_BuildValue("(O(O))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); } static PyMethodDef os_sched_param_reduce_method = { diff --git a/Python/bytecodes.c b/Python/bytecodes.c index f9f14322df0a5e..6ebd9ebdfce1bb 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -11,7 +11,6 @@ #include "pycore_audit.h" // _PySys_Audit() #include "pycore_backoff.h" #include "pycore_cell.h" // PyCell_GetRef() -#include "pycore_ceval.h" #include "pycore_code.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS #include "pycore_function.h" @@ -5362,10 +5361,6 @@ dummy_func( tier2 op(_MAKE_WARM, (--)) { current_executor->vm_data.warm = true; - // It's okay if this ends up going negative. - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } } tier2 op(_FATAL_ERROR, (--)) { diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 6bf64868cbb2d3..9b6506ac3326b3 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1398,7 +1398,7 @@ _Py_HandlePending(PyThreadState *tstate) if ((breaker & _PY_EVAL_JIT_INVALIDATE_COLD_BIT) != 0) { _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); _Py_Executors_InvalidateCold(tstate->interp); - tstate->interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + tstate->interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; } /* GIL drop request */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 4ed03b7fb01bdf..8083913b1a1cfc 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -358,7 +358,9 @@ do { \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ DISPATCH(); \ diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0e4d86463761a0..9ce0a9f8a4d87b 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -7409,9 +7409,6 @@ case _MAKE_WARM: { current_executor->vm_data.warm = true; - if (--tstate->interp->trace_run_counter == 0) { - _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); - } break; } diff --git a/Python/optimizer.c b/Python/optimizer.c index 6ad9124744859a..f44f8a9614b846 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -6,6 +6,7 @@ #include "pycore_interp.h" #include "pycore_backoff.h" #include "pycore_bitutils.h" // _Py_popcount32() +#include "pycore_ceval.h" // _Py_set_eval_breaker_bit #include "pycore_code.h" // _Py_GetBaseCodeUnit #include "pycore_function.h" // _PyFunction_LookupByVersion() #include "pycore_interpframe.h" @@ -1343,6 +1344,14 @@ uop_optimize( return -1; } assert(length <= UOP_MAX_TRACE_LENGTH); + + // Check executor coldness + PyThreadState *tstate = PyThreadState_Get(); + // It's okay if this ends up going negative. + if (--tstate->interp->executor_creation_counter == 0) { + _Py_set_eval_breaker_bit(tstate, _PY_EVAL_JIT_INVALIDATE_COLD_BIT); + } + *exec_ptr = executor; return 1; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index b813166f167d70..8fcb31cfd12299 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -506,6 +506,7 @@ pycore_init_runtime(_PyRuntimeState *runtime, _PyRuntimeState_SetFinalizing(runtime, NULL); _Py_InitVersion(); + _Py_DumpTraceback_Init(); status = _Py_HashRandomization_Init(config); if (_PyStatus_EXCEPTION(status)) { diff --git a/Python/pystate.c b/Python/pystate.c index 5d0927c6c08196..2141e842a37d2f 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -571,7 +571,7 @@ init_interpreter(PyInterpreterState *interp, interp->executor_list_head = NULL; interp->executor_deletion_list_head = NULL; interp->executor_deletion_list_remaining_capacity = 0; - interp->trace_run_counter = JIT_CLEANUP_THRESHOLD; + interp->executor_creation_counter = JIT_CLEANUP_THRESHOLD; if (interp != &runtime->_main_interpreter) { /* Fix the self-referential, statically initialized fields. */ interp->dtoa = (struct _dtoa_state)_dtoa_state_INIT(interp); diff --git a/Python/traceback.c b/Python/traceback.c index 46106e52dbaf82..2639d50aa66717 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -69,6 +69,13 @@ class traceback "PyTracebackObject *" "&PyTraceback_Type" #include "clinic/traceback.c.h" + +#ifdef MS_WINDOWS +typedef HRESULT (WINAPI *PF_GET_THREAD_DESCRIPTION)(HANDLE, PCWSTR*); +static PF_GET_THREAD_DESCRIPTION pGetThreadDescription = NULL; +#endif + + static PyObject * tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, int lineno) @@ -1107,23 +1114,12 @@ _Py_DumpTraceback(int fd, PyThreadState *tstate) # endif #endif -/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if - is_current is true, "Thread 0xHHHH:\n" otherwise. - - This function is signal safe. */ +// Write the thread name static void -write_thread_id(int fd, PyThreadState *tstate, int is_current) +write_thread_name(int fd, PyThreadState *tstate) { - if (is_current) - PUTS(fd, "Current thread 0x"); - else - PUTS(fd, "Thread 0x"); - _Py_DumpHexadecimal(fd, - tstate->thread_id, - sizeof(unsigned long) * 2); - - // Write the thread name +#ifndef MS_WINDOWS #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) char name[100]; pthread_t thread = (pthread_t)tstate->thread_id; @@ -1142,6 +1138,54 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) } } #endif +#else + // Windows implementation + if (pGetThreadDescription == NULL) { + return; + } + + HANDLE thread = OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, tstate->thread_id); + if (thread == NULL) { + return; + } + + wchar_t *wname; + HRESULT hr = pGetThreadDescription(thread, &wname); + if (!FAILED(hr)) { + char *name = _Py_EncodeLocaleRaw(wname, NULL); + if (name != NULL) { + size_t len = strlen(name); + if (len) { + PUTS(fd, " ["); + (void)_Py_write_noraise(fd, name, len); + PUTS(fd, "]"); + } + PyMem_RawFree(name); + } + LocalFree(wname); + } + CloseHandle(thread); +#endif +} + + +/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if + is_current is true, "Thread 0xHHHH:\n" otherwise. + + This function is signal safe (except on Windows). */ + +static void +write_thread_id(int fd, PyThreadState *tstate, int is_current) +{ + if (is_current) + PUTS(fd, "Current thread 0x"); + else + PUTS(fd, "Thread 0x"); + _Py_DumpHexadecimal(fd, + tstate->thread_id, + sizeof(unsigned long) * 2); + + write_thread_name(fd, tstate); PUTS(fd, " (most recent call first):\n"); } @@ -1336,3 +1380,20 @@ _Py_InitDumpStack(void) (void)backtrace(callstack, 1); #endif } + + +void +_Py_DumpTraceback_Init(void) +{ +#ifdef MS_WINDOWS + if (pGetThreadDescription != NULL) { + return; + } + + HMODULE kernelbase = GetModuleHandleW(L"kernelbase.dll"); + if (kernelbase != NULL) { + pGetThreadDescription = (PF_GET_THREAD_DESCRIPTION)GetProcAddress( + kernelbase, "GetThreadDescription"); + } +#endif +}