From d971dd5176daa328446c15bf89f0e3672c885e7e Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 09:09:14 +0800 Subject: [PATCH 1/8] Fix: Better module info for SyntaxWarnings during AST parsing --- Include/internal/pycore_pyerrors.h | 2 + Python/_warnings.c | 69 +++++++++++++++++++++--------- Python/compile.c | 4 +- Python/errors.c | 25 ++++++++++- 4 files changed, 76 insertions(+), 24 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2c2048f7e1272a..1068f164dc21a0 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -124,6 +124,8 @@ extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); +extern int _PyErr_EmitSyntaxWarningFromCompiler(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Python/_warnings.c b/Python/_warnings.c index 12e6172b0cf828..86f012cb6d73ec 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -616,33 +616,60 @@ already_warned(PyInterpreterState *interp, PyObject *registry, PyObject *key, static PyObject * normalize_module(PyObject *filename) { - PyObject *module; - int kind; - const void *data; - Py_ssize_t len; - - len = PyUnicode_GetLength(filename); - if (len < 0) + PyObject *module_name = NULL; + PyObject *os_path = NULL; + PyObject *basename = NULL; + PyObject *splitext = NULL; + PyObject *root = NULL; + + os_path = PyImport_ImportModule("os.path"); + if (os_path == NULL) { return NULL; + } - if (len == 0) - return PyUnicode_FromString(""); + basename = PyObject_CallMethod(os_path, "basename", "O", filename); + if (basename == NULL) { + goto cleanup; + } - kind = PyUnicode_KIND(filename); - data = PyUnicode_DATA(filename); + splitext = PyObject_CallMethod(os_path, "splitext", "O", basename); + if (splitext == NULL) { + goto cleanup; + } - /* if filename.endswith(".py"): */ - if (len >= 3 && - PyUnicode_READ(kind, data, len-3) == '.' && - PyUnicode_READ(kind, data, len-2) == 'p' && - PyUnicode_READ(kind, data, len-1) == 'y') - { - module = PyUnicode_Substring(filename, 0, len-3); + root = PyTuple_GetItem(splitext, 0); + if (root == NULL) { + goto cleanup; } - else { - module = Py_NewRef(filename); + + if (PyUnicode_CompareWithASCIIString(root, "__init__") == 0) { + PyObject *dirname = PyObject_CallMethod(os_path, "dirname", "O", filename); + if (dirname == NULL) { + goto cleanup; + } + module_name = PyObject_CallMethod(os_path, "basename", "O", dirname); + Py_DECREF(dirname); + } else { + module_name = Py_NewRef(root); } - return module; + +cleanup: + Py_XDECREF(os_path); + Py_XDECREF(basename); + Py_XDECREF(splitext); + + if (module_name == NULL) { + // Fallback or error occurred + PyErr_Clear(); + return PyUnicode_FromString(""); + } + + if (PyUnicode_GetLength(module_name) == 0) { + Py_DECREF(module_name); + return PyUnicode_FromString(""); + } + + return module_name; } static int diff --git a/Python/compile.c b/Python/compile.c index c04391e682f9ac..7675072fd5d1c1 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1210,8 +1210,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) if (msg == NULL) { return ERROR; } - int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + int ret = _PyErr_EmitSyntaxWarningFromCompiler(msg, c->c_filename, loc.lineno, loc.col_offset + 1, + loc.end_lineno, loc.end_col_offset + 1); Py_DECREF(msg); return ret; } diff --git a/Python/errors.c b/Python/errors.c index a3122f76bdd87d..919ac43b3d1878 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1,4 +1,3 @@ - /* Error handling */ #include "Python.h" @@ -1952,6 +1951,30 @@ _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_ return 0; } +/* Emits a SyntaxWarning and returns 0 on success. + If a SyntaxWarning is raised as error, replaces it with a SyntaxError + and returns -1. + This version is for the compiler, and derives the module from the filename. +*/ +int +_PyErr_EmitSyntaxWarningFromCompiler(PyObject *msg, PyObject *filename, int lineno, int col_offset, + int end_lineno, int end_col_offset) +{ + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, + filename, lineno, NULL, NULL) < 0) + { + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + /* Replace the SyntaxWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + _PyErr_RaiseSyntaxError(msg, filename, lineno, col_offset, + end_lineno, end_col_offset); + } + return -1; + } + return 0; +} + /* Attempt to load the line of text that the exception refers to. If it fails, it will return NULL but will not set an exception. From 0deed9eccbd58723a25a7dca4555729a08893618 Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 09:48:39 +0800 Subject: [PATCH 2/8] Fix SyntaxWarning deduplication with pseudo-filenames --- Python/errors.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Python/errors.c b/Python/errors.c index 919ac43b3d1878..75fd68eb836885 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1960,6 +1960,15 @@ int _PyErr_EmitSyntaxWarningFromCompiler(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) { + Py_ssize_t len = PyUnicode_GET_LENGTH(filename); + if (len > 1 && + PyUnicode_READ_CHAR(filename, 0) == '<' && + PyUnicode_READ_CHAR(filename, len - 1) == '>') + { + return _PyErr_EmitSyntaxWarning(msg, filename, lineno, col_offset, + end_lineno, end_col_offset); + } + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, NULL, NULL) < 0) { From 4b056295fa6f622434e94c6c472f6ded42b4014f Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 11:05:08 +0800 Subject: [PATCH 3/8] Fix SyntaxWarning deduplication with pseudo-filenames --- .../2024-06-23-10-56-32.gh-issue-135801.fix-warning-dedup.rst | 1 + Python/_warnings.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2024-06-23-10-56-32.gh-issue-135801.fix-warning-dedup.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-23-10-56-32.gh-issue-135801.fix-warning-dedup.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-23-10-56-32.gh-issue-135801.fix-warning-dedup.rst new file mode 100644 index 00000000000000..831d0edcfea6d9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-23-10-56-32.gh-issue-135801.fix-warning-dedup.rst @@ -0,0 +1 @@ +Fix warning deduplication regression for SyntaxWarnings with pseudo-filenames (e.g., , ). diff --git a/Python/_warnings.c b/Python/_warnings.c index 86f012cb6d73ec..3a53ce297c6f74 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -663,7 +663,7 @@ normalize_module(PyObject *filename) PyErr_Clear(); return PyUnicode_FromString(""); } - + if (PyUnicode_GetLength(module_name) == 0) { Py_DECREF(module_name); return PyUnicode_FromString(""); From 423fae39d981113da2ac5e40662e660f85513fc4 Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 14:08:10 +0800 Subject: [PATCH 4/8] Add Unit Test --- Lib/test/test_warnings/__init__.py | 35 ++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 5c3b1250ceb045..82f458dd5f2ba0 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -771,15 +771,32 @@ def test_improper_input(self): self.assertRaises(UserWarning, self.module.warn, 'convert to error') def test_import_from_module(self): - with self.module.catch_warnings(): - self.module._setoption('ignore::Warning') - with self.assertRaises(self.module._OptionError): - self.module._setoption('ignore::TestWarning') - with self.assertRaises(self.module._OptionError): - self.module._setoption('ignore::test.test_warnings.bogus') - self.module._setoption('error::test.test_warnings.TestWarning') - with self.assertRaises(TestWarning): - self.module.warn('test warning', TestWarning) + script = support.script_helper.make_script('test_warnings_importer', + 'import test.test_warnings.data.import_warning') + rc, out, err = assert_python_ok(script) + self.assertNotIn(b'UserWarning', err) + + def test_syntax_warning_for_compiler(self): + # Test that SyntaxWarning from the compiler has a proper module name, + # not a guessed one like 'sys'. gh-135801 + code = textwrap.dedent("""\ + class A: + def func(self): + return self.var is 2 + """) + # The name of the script is 'test_sw' + with os_helper.temp_dir() as script_dir: + script_name = support.script_helper.make_script(script_dir, 'test_sw', code) + # We want to check that the warning filter for 'test_sw' module works. + rc, out, err = assert_python_failure("-W", "error::SyntaxWarning:test_sw", + script_name) + self.assertEqual(rc, 1) + self.assertEqual(out, b'') + # Check that we got a SyntaxError. + err = err.decode() + self.assertIn("""SyntaxError: "is" with 'int' literal. Did you mean "=="?""", err) + # Check that the filename in the traceback is correct. + self.assertIn(os.path.basename(script_name), err) class CWCmdLineTests(WCmdLineTests, unittest.TestCase): From 20c4390fb76ba946b706bf056dffc668ed14682b Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 14:17:16 +0800 Subject: [PATCH 5/8] Fix test_warnings failures in CI --- Lib/test/test_warnings/__init__.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 82f458dd5f2ba0..075be56432fa0c 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -771,9 +771,11 @@ def test_improper_input(self): self.assertRaises(UserWarning, self.module.warn, 'convert to error') def test_import_from_module(self): - script = support.script_helper.make_script('test_warnings_importer', - 'import test.test_warnings.data.import_warning') - rc, out, err = assert_python_ok(script) + with support.temp_dir() as script_dir: + script = support.script_helper.make_script(script_dir, + 'test_warnings_importer', + 'import test.test_warnings.data.import_warning') + rc, out, err = assert_python_ok(script) self.assertNotIn(b'UserWarning', err) def test_syntax_warning_for_compiler(self): @@ -794,6 +796,7 @@ def func(self): self.assertEqual(out, b'') # Check that we got a SyntaxError. err = err.decode() + err = support.strip_ansi(err) self.assertIn("""SyntaxError: "is" with 'int' literal. Did you mean "=="?""", err) # Check that the filename in the traceback is correct. self.assertIn(os.path.basename(script_name), err) From 3ddfa7173696a36e30ffe22493817e9e4685d310 Mon Sep 17 00:00:00 2001 From: heliang Date: Mon, 23 Jun 2025 15:25:53 +0800 Subject: [PATCH 6/8] Fix: Correct ImportError and NameError in test_warnings --- Lib/test/test_warnings/__init__.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 075be56432fa0c..c11b695f01b8cb 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -14,6 +14,8 @@ from test.support import import_helper from test.support import os_helper from test.support import warnings_helper +from test.support import script_helper +from _pyrepl.completing_reader import stripcolor as strip_ansi from test.support import force_not_colorized from test.support.script_helper import assert_python_ok, assert_python_failure @@ -771,8 +773,8 @@ def test_improper_input(self): self.assertRaises(UserWarning, self.module.warn, 'convert to error') def test_import_from_module(self): - with support.temp_dir() as script_dir: - script = support.script_helper.make_script(script_dir, + with os_helper.temp_dir() as script_dir: + script = script_helper.make_script(script_dir, 'test_warnings_importer', 'import test.test_warnings.data.import_warning') rc, out, err = assert_python_ok(script) @@ -788,7 +790,7 @@ def func(self): """) # The name of the script is 'test_sw' with os_helper.temp_dir() as script_dir: - script_name = support.script_helper.make_script(script_dir, 'test_sw', code) + script_name = script_helper.make_script(script_dir, 'test_sw', code) # We want to check that the warning filter for 'test_sw' module works. rc, out, err = assert_python_failure("-W", "error::SyntaxWarning:test_sw", script_name) @@ -796,7 +798,7 @@ def func(self): self.assertEqual(out, b'') # Check that we got a SyntaxError. err = err.decode() - err = support.strip_ansi(err) + err = strip_ansi(err) self.assertIn("""SyntaxError: "is" with 'int' literal. Did you mean "=="?""", err) # Check that the filename in the traceback is correct. self.assertIn(os.path.basename(script_name), err) From 8c9913e608a1f03c7e9724fd82cd30c95e916780 Mon Sep 17 00:00:00 2001 From: heliang <3596006474@qq.com> Date: Mon, 30 Jun 2025 08:59:39 +0800 Subject: [PATCH 7/8] Fix improves SyntaxWarning module information by directly deriving module names from filenames rather than the call stack, and streamlines warning emission. --- Include/internal/pycore_pyerrors.h | 2 -- Python/compile.c | 4 +-- Python/errors.c | 41 ++++++++++++------------------ 3 files changed, 18 insertions(+), 29 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 1068f164dc21a0..2c2048f7e1272a 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -124,8 +124,6 @@ extern PyObject* _PyErr_NoMemory(PyThreadState *tstate); extern int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); -extern int _PyErr_EmitSyntaxWarningFromCompiler(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset); extern void _PyErr_RaiseSyntaxError(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset); diff --git a/Python/compile.c b/Python/compile.c index 7675072fd5d1c1..c04391e682f9ac 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -1210,8 +1210,8 @@ _PyCompile_Warn(compiler *c, location loc, const char *format, ...) if (msg == NULL) { return ERROR; } - int ret = _PyErr_EmitSyntaxWarningFromCompiler(msg, c->c_filename, loc.lineno, loc.col_offset + 1, - loc.end_lineno, loc.end_col_offset + 1); + int ret = _PyErr_EmitSyntaxWarning(msg, c->c_filename, loc.lineno, loc.col_offset + 1, + loc.end_lineno, loc.end_col_offset + 1); Py_DECREF(msg); return ret; } diff --git a/Python/errors.c b/Python/errors.c index 75fd68eb836885..7eb69378f972c1 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -1936,39 +1936,30 @@ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) { - if (_PyErr_WarnExplicitObjectWithContext(PyExc_SyntaxWarning, msg, - filename, lineno) < 0) - { - if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { - /* Replace the SyntaxWarning exception with a SyntaxError - to get a more accurate error report */ - PyErr_Clear(); - _PyErr_RaiseSyntaxError(msg, filename, lineno, col_offset, - end_lineno, end_col_offset); - } - return -1; - } - return 0; -} - -/* Emits a SyntaxWarning and returns 0 on success. - If a SyntaxWarning is raised as error, replaces it with a SyntaxError - and returns -1. - This version is for the compiler, and derives the module from the filename. -*/ -int -_PyErr_EmitSyntaxWarningFromCompiler(PyObject *msg, PyObject *filename, int lineno, int col_offset, - int end_lineno, int end_col_offset) -{ + /* For pseudo-filenames (e.g., , ), use the original approach + to maintain compatibility with existing behavior */ Py_ssize_t len = PyUnicode_GET_LENGTH(filename); if (len > 1 && PyUnicode_READ_CHAR(filename, 0) == '<' && PyUnicode_READ_CHAR(filename, len - 1) == '>') { - return _PyErr_EmitSyntaxWarning(msg, filename, lineno, col_offset, + if (_PyErr_WarnExplicitObjectWithContext(PyExc_SyntaxWarning, msg, + filename, lineno) < 0) + { + if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { + /* Replace the SyntaxWarning exception with a SyntaxError + to get a more accurate error report */ + PyErr_Clear(); + _PyErr_RaiseSyntaxError(msg, filename, lineno, col_offset, end_lineno, end_col_offset); + } + return -1; + } + return 0; } + /* For regular files, derive the module from the filename by passing NULL + as the module argument to PyErr_WarnExplicitObject */ if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, filename, lineno, NULL, NULL) < 0) { From de3391ffb3b7763a674a4a94a1d8496525d583f2 Mon Sep 17 00:00:00 2001 From: heliang <3596006474@qq.com> Date: Tue, 1 Jul 2025 09:54:48 +0800 Subject: [PATCH 8/8] Optimize asyncio.to_thread to avoid contextvars.copy_context() overhead for empty contexts --- Lib/asyncio/threads.py | 7 +++-- Lib/test/test_asyncio/test_threads.py | 38 ++++++++++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de16..b1b030bf2c0421 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -21,5 +21,8 @@ async def to_thread(func, /, *args, **kwargs): """ loop = events.get_running_loop() ctx = contextvars.copy_context() - func_call = functools.partial(ctx.run, func, *args, **kwargs) - return await loop.run_in_executor(None, func_call) + if len(ctx) == 0: + callback = functools.partial(func, *args, **kwargs) + else: + callback = functools.partial(ctx.run, func, *args, **kwargs) + return await loop.run_in_executor(None, callback) diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index c98c9a9b395ff9..a1323c4dbdbb21 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -2,13 +2,14 @@ import asyncio import unittest +import functools from contextvars import ContextVar from unittest import mock def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): @@ -61,6 +62,41 @@ def get_ctx(): self.assertEqual(result, 'parrot') + @mock.patch('asyncio.base_events.BaseEventLoop.run_in_executor') + async def test_to_thread_optimization_path(self, run_in_executor): + # This test ensures that `to_thread` uses the correct execution path + # based on whether the context is empty or not. + + # `to_thread` awaits the future returned by `run_in_executor`. + # We need to provide a completed future as a return value for the mock. + fut = asyncio.Future() + fut.set_result(None) + run_in_executor.return_value = fut + + def myfunc(): + pass + + # Test with an empty context (optimized path) + await asyncio.to_thread(myfunc) + run_in_executor.assert_called_once() + + callback = run_in_executor.call_args.args[1] + self.assertIsInstance(callback, functools.partial) + self.assertIs(callback.func, myfunc) + run_in_executor.reset_mock() + + # Test with a non-empty context (standard path) + var = ContextVar('var') + var.set('value') + + await asyncio.to_thread(myfunc) + run_in_executor.assert_called_once() + + callback = run_in_executor.call_args.args[1] + self.assertIsInstance(callback, functools.partial) + self.assertIsNot(callback.func, myfunc) # Should be ctx.run + self.assertIs(callback.args[0], myfunc) + if __name__ == "__main__": unittest.main()