From fc0c9c24121c1a62b25b79062286f976699b59e9 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 May 2025 19:58:34 +0300 Subject: [PATCH 1/6] gh-133454: Reduce the number of threads in test_racing_getbuf_and_releasebuf (GH-133458) The original reproducer only used 10 threads. --- Lib/test/test_memoryview.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 61b068c630c7ce..64f440f180bbf0 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -743,19 +743,21 @@ def test_racing_getbuf_and_releasebuf(self): from multiprocessing.managers import SharedMemoryManager except ImportError: self.skipTest("Test requires multiprocessing") - from threading import Thread + from threading import Thread, Event - n = 100 + start = Event() with SharedMemoryManager() as smm: obj = smm.ShareableList(range(100)) - threads = [] - for _ in range(n): - # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` - # counter of `memoryview`, this issue has no direct relation with `ShareableList`. - threads.append(Thread(target=obj.count, args=(1,))) - + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] with threading_helper.start_threads(threads): - pass + start.set() del obj From 77eade39f972a4f3d8e9fec00288779f35ceee21 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 May 2025 19:59:10 +0300 Subject: [PATCH 2/6] gh-134578: Mark more slow tests (GH-134579) --- Lib/test/test_ast/test_ast.py | 1 + Lib/test/test_capi/test_object.py | 1 + Lib/test/test_collections.py | 2 ++ Lib/test/test_json/test_recursion.py | 1 + Lib/test/test_statistics.py | 1 + 5 files changed, 6 insertions(+) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 1479a8eafd62ec..46745cfa8f8325 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3292,6 +3292,7 @@ def check_output(self, source, expect, *flags): expect = self.text_normalize(expect) self.assertEqual(res, expect) + @support.requires_resource('cpu') def test_invocation(self): # test various combinations of parameters base_flags = ( diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index cd772120bdee2b..d4056727d07fbf 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -221,6 +221,7 @@ def test_decref_freed_object(self): """ self.check_negative_refcount(code) + @support.requires_resource('cpu') def test_decref_delayed(self): # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path # handles destructors that are possibly re-entrant or trigger a GC. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1e93530398be79..d9d61e5c2053e3 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -542,6 +542,8 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) + @support.requires_resource('cpu') + def test_large_size(self): n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 8f0e5e078ed0d4..5d7b56ff9ad285 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -86,6 +86,7 @@ def test_highly_nested_objects_decoding(self): @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 5980f939185965..0dd619dd7c8ceb 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2346,6 +2346,7 @@ def test_mixed_int_and_float(self): class TestKDE(unittest.TestCase): + @support.requires_resource('cpu') def test_kde(self): kde = statistics.kde StatisticsError = statistics.StatisticsError From 393773ae8763202ecf7afff81c4d57bd37c62ff6 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Fri, 23 May 2025 21:07:49 +0300 Subject: [PATCH 3/6] gh-134565: Use ExceptionGroup to handle multiple errors in unittest.doModuleCleanups() (GH-134566) --- Lib/test/test_unittest/test_result.py | 37 +++++++++++-- Lib/test/test_unittest/test_runner.py | 52 +++++++++++++++++-- Lib/unittest/case.py | 4 +- Lib/unittest/suite.py | 20 +++++-- ...-05-23-10-15-36.gh-issue-134565.zmb66C.rst | 3 ++ 5 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-23-10-15-36.gh-issue-134565.zmb66C.rst diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 9ac4c52449c2ff..3f44e617303f81 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1282,14 +1282,22 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 1) + self.assertEqual(len(result.errors), 2) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[1] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferSetUpModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1313,22 +1321,34 @@ def setUpModule(): suite(result) expected_out = '\nStdout:\nset up module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'setUpModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\nset up module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertIn(expected_out, formatted_exc) self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertIn(expected_out, formatted_exc) + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + def testBufferTearDownModule_DoModuleCleanups(self): with captured_stdout() as stdout: result = unittest.TestResult() @@ -1355,21 +1375,32 @@ def tearDownModule(): suite(result) expected_out = '\nStdout:\ntear down module\ndo cleanup2\ndo cleanup1\n' self.assertEqual(stdout.getvalue(), expected_out) - self.assertEqual(len(result.errors), 2) + self.assertEqual(len(result.errors), 3) description = 'tearDownModule (Module)' test_case, formatted_exc = result.errors[0] self.assertEqual(test_case.description, description) self.assertIn('ZeroDivisionError: division by zero', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ValueError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn('\nStdout:\ntear down module\n', formatted_exc) + test_case, formatted_exc = result.errors[1] self.assertEqual(test_case.description, description) self.assertIn('ValueError: bad cleanup2', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) self.assertNotIn('ZeroDivisionError', formatted_exc) self.assertNotIn('TypeError', formatted_exc) self.assertIn(expected_out, formatted_exc) + test_case, formatted_exc = result.errors[2] + self.assertEqual(test_case.description, description) + self.assertIn('TypeError: bad cleanup1', formatted_exc) + self.assertNotIn('ExceptionGroup', formatted_exc) + self.assertNotIn('ZeroDivisionError', formatted_exc) + self.assertNotIn('ValueError', formatted_exc) + self.assertIn(expected_out, formatted_exc) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 4d3cfd60b8d9c3..a47e2ebb59da02 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -13,6 +13,7 @@ LoggingResult, ResultWithNoStartTestRunStopTestRun, ) +from test.support.testcase import ExceptionIsLikeMixin def resultFactory(*_): @@ -604,7 +605,7 @@ class EmptyTest(unittest.TestCase): @support.force_not_colorized_test_class -class TestModuleCleanUp(unittest.TestCase): +class TestModuleCleanUp(ExceptionIsLikeMixin, unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -646,11 +647,50 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(CustomError) as e: + with self.assertRaises(Exception) as e: unittest.case.doModuleCleanups() - self.assertEqual(str(e.exception), 'CleanUpExc') + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(unittest.case._module_cleanups, []) + def test_doModuleCleanup_with_multiple_errors_in_addModuleCleanup(self): + def module_cleanup_bad1(): + raise TypeError('CleanUpExc1') + + def module_cleanup_bad2(): + raise ValueError('CleanUpExc2') + + class Module: + unittest.addModuleCleanup(module_cleanup_bad1) + unittest.addModuleCleanup(module_cleanup_bad2) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ])) + + def test_doModuleCleanup_with_exception_group_in_addModuleCleanup(self): + def module_cleanup_bad(): + raise ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]) + + class Module: + unittest.addModuleCleanup(module_cleanup_bad) + with self.assertRaises(ExceptionGroup) as e: + unittest.case.doModuleCleanups() + self.assertExceptionIsLike(e.exception, + ExceptionGroup('module cleanup failed', [ + ExceptionGroup('CleanUpExc', [ + ValueError('CleanUpExc2'), + TypeError('CleanUpExc1'), + ]), + ])) + def test_addModuleCleanup_arg_errors(self): cleanups = [] def cleanup(*args, **kwargs): @@ -871,9 +911,11 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(CustomError) as cm: + with self.assertRaises(Exception) as cm: suite.debug() - self.assertEqual(str(cm.exception), 'CleanUpExc') + self.assertExceptionIsLike(cm.exception, + ExceptionGroup('module cleanup failed', + [CustomError('CleanUpExc')])) self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) self.assertEqual(unittest.case._module_cleanups, []) diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index 884fc1b21f64d8..db10de68e4ac73 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -149,9 +149,7 @@ def doModuleCleanups(): except Exception as exc: exceptions.append(exc) if exceptions: - # Swallows all but first exception. If a multi-exception handler - # gets written we should use that here instead. - raise exceptions[0] + raise ExceptionGroup('module cleanup failed', exceptions) def skip(reason): diff --git a/Lib/unittest/suite.py b/Lib/unittest/suite.py index 6f45b6fe5f6039..ae9ca2d615de06 100644 --- a/Lib/unittest/suite.py +++ b/Lib/unittest/suite.py @@ -223,6 +223,11 @@ def _handleModuleFixture(self, test, result): if result._moduleSetUpFailed: try: case.doModuleCleanups() + except ExceptionGroup as eg: + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'setUpModule', + currentModule) except Exception as e: self._createClassOrModuleLevelException(result, e, 'setUpModule', @@ -235,15 +240,15 @@ def _createClassOrModuleLevelException(self, result, exc, method_name, errorName = f'{method_name} ({parent})' self._addClassOrModuleLevelException(result, exc, errorName, info) - def _addClassOrModuleLevelException(self, result, exception, errorName, + def _addClassOrModuleLevelException(self, result, exc, errorName, info=None): error = _ErrorHolder(errorName) addSkip = getattr(result, 'addSkip', None) - if addSkip is not None and isinstance(exception, case.SkipTest): - addSkip(error, str(exception)) + if addSkip is not None and isinstance(exc, case.SkipTest): + addSkip(error, str(exc)) else: if not info: - result.addError(error, sys.exc_info()) + result.addError(error, (type(exc), exc, exc.__traceback__)) else: result.addError(error, info) @@ -273,6 +278,13 @@ def _handleModuleTearDown(self, result): previousModule) try: case.doModuleCleanups() + except ExceptionGroup as eg: + if isinstance(result, _DebugResult): + raise + for e in eg.exceptions: + self._createClassOrModuleLevelException(result, e, + 'tearDownModule', + previousModule) except Exception as e: if isinstance(result, _DebugResult): raise diff --git a/Misc/NEWS.d/next/Library/2025-05-23-10-15-36.gh-issue-134565.zmb66C.rst b/Misc/NEWS.d/next/Library/2025-05-23-10-15-36.gh-issue-134565.zmb66C.rst new file mode 100644 index 00000000000000..17d2b23b62d3c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-23-10-15-36.gh-issue-134565.zmb66C.rst @@ -0,0 +1,3 @@ +:func:`unittest.doModuleCleanups` no longer swallows all but first exception +raised in the cleanup code, but raises a :exc:`ExceptionGroup` if multiple +errors occurred. From 05a19b5e56894fd1e63aff6b38fb23ad7c7b3047 Mon Sep 17 00:00:00 2001 From: Daniel Li Date: Fri, 23 May 2025 14:45:45 -0400 Subject: [PATCH 4/6] gh-120170: Exclude __mp_main__ in C version of whichmodule() (#120171) Co-authored-by: Lysandros Nikolaou --- .../Library/2024-06-06-17-49-07.gh-issue-120170.DUxhmT.rst | 3 +++ Modules/_pickle.c | 4 ++++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-06-06-17-49-07.gh-issue-120170.DUxhmT.rst diff --git a/Misc/NEWS.d/next/Library/2024-06-06-17-49-07.gh-issue-120170.DUxhmT.rst b/Misc/NEWS.d/next/Library/2024-06-06-17-49-07.gh-issue-120170.DUxhmT.rst new file mode 100644 index 00000000000000..ce7d874aa20375 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-06-17-49-07.gh-issue-120170.DUxhmT.rst @@ -0,0 +1,3 @@ +Fix an issue in the :mod:`!_pickle` extension module in which importing +:mod:`multiprocessing` could change how pickle identifies which module an +object belongs to, potentially breaking the unpickling of those objects. diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d260f1a68f8c70..29ef0cb0c2e088 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -1879,6 +1879,10 @@ _checkmodule(PyObject *module_name, PyObject *module, _PyUnicode_EqualToASCIIString(module_name, "__main__")) { return -1; } + if (PyUnicode_Check(module_name) && + _PyUnicode_EqualToASCIIString(module_name, "__mp_main__")) { + return -1; + } PyObject *candidate = getattribute(module, dotted_path, 0); if (candidate == NULL) { From 9a2346df861f26d5f8d054ad2f9c37134dee3822 Mon Sep 17 00:00:00 2001 From: "Jiucheng(Oliver)" Date: Fri, 23 May 2025 15:22:14 -0400 Subject: [PATCH 5/6] gh-134381: Fix RuntimeError when starting not-yet started Thread after fork (gh-134514) --- Lib/test/_test_multiprocessing.py | 22 +++++++++++++++++++ ...-05-22-14-48-19.gh-issue-134381.2BXhth.rst | 1 + Modules/_threadmodule.c | 6 +++++ 3 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 6a20a1eb03e32b..75f31d858d3306 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6844,6 +6844,28 @@ def f(x): return x*x self.assertEqual("332833500", out.decode('utf-8').strip()) self.assertFalse(err, msg=err.decode('utf-8')) + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + # # Mixins diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst new file mode 100644 index 00000000000000..aa8900296ae2fc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst @@ -0,0 +1 @@ +Fix :exc:`RuntimeError` when using a not-started :class:`threading.Thread` after calling :func:`os.fork` diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 74b972b201a546..cc83be4b5ff311 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -296,6 +296,12 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) continue; } + // Keep handles for threads that have not been started yet. They are + // safe to start in the child process. + if (handle->state == THREAD_HANDLE_NOT_STARTED) { + continue; + } + // Mark all threads as done. Any attempts to join or detach the // underlying OS thread (if any) could crash. We are the only thread; // it's safe to set this non-atomically. From 8a793c4a365d06a7264887698ccd7d6ba6aba9f2 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 23 May 2025 14:04:20 -0600 Subject: [PATCH 6/6] gh-134557: Revert "gh-132775: Use _PyCode GetScriptXIData()" (gh-134599) This reverts commit 09e72cf091d, AKA gh-134511. We are reverting due to refleaks on free-threaded builds. --- Lib/test/support/interpreters/channels.py | 2 +- Lib/test/test__interpreters.py | 25 +- Lib/test/test_interpreters/test_api.py | 23 +- Modules/_interpretersmodule.c | 294 ++++++++++++++++------ 4 files changed, 240 insertions(+), 104 deletions(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 3b6e0f0effd969..7a2bd7d63f808f 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -69,7 +69,7 @@ def list_all(): if not hasattr(send, '_unboundop'): send._set_unbound(unboundop) else: - assert send._unbound[0] == unboundop + assert send._unbound[0] == op channels.append(chan) return channels diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index ad3ebbfdff64a7..63fdaad8de7ef5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,15 +474,13 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = r'_interpreters.exec\(\) argument 3 must be dict, not int' + msg = "expected 'shared' to be a dict" with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) - msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) - msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) @@ -954,8 +952,7 @@ def test_invalid_syntax(self): """) with self.subTest('script'): - with self.assertRaises(SyntaxError): - _interpreters.run_string(self.id, script) + self.assert_run_failed(SyntaxError, script) with self.subTest('module'): modname = 'spam_spam_spam' @@ -1022,19 +1019,12 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - failed = None def f(): - nonlocal failed - try: - _interpreters.set___main___attrs(self.id, dict(w=w)) - _interpreters.run_func(self.id, script) - except Exception as exc: - failed = exc + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() - if failed: - raise Exception from failed with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1063,16 +1053,19 @@ def test_closure(self): spam = True def script(): assert spam - with self.assertRaises(ValueError): + + with self.assertRaises(TypeError): _interpreters.run_func(self.id, script) + # XXX This hasn't been fixed yet. + @unittest.expectedFailure def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) -# @unittest.skip("we're not quite there yet") + @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 165949167ceba8..1e2d572b1cbb81 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -839,16 +839,9 @@ def test_bad_script(self): interp.exec(10) def test_bytes_for_script(self): - r, w = self.pipe() - RAN = b'R' - DONE = b'D' interp = interpreters.create() - interp.exec(f"""if True: - import os - os.write({w}, {RAN!r}) - """) - os.write(w, DONE) - self.assertEqual(os.read(r, 1), RAN) + with self.assertRaises(TypeError): + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -1017,6 +1010,8 @@ def test_call(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1041,8 +1036,6 @@ def test_call(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): with self.assertRaises(Exception): @@ -1058,6 +1051,8 @@ def test_call_in_thread(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1084,8 +1079,6 @@ def test_call_in_thread(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): if args or kwargs: @@ -1625,8 +1618,8 @@ def test_exec(self): def test_call(self): with self.subTest('no args'): interpid = _interpreters.create() - with self.assertRaises(ValueError): - _interpreters.call(interpid, call_func_return_shareable) + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) with self.subTest('uncaught exception'): interpid = _interpreters.create() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 376517ab92360f..f4807fd214b19e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -9,6 +9,7 @@ #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_function.h" // _PyFunction_VerifyStateless() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -360,6 +361,81 @@ _get_current_xibufferview_type(void) } +/* Python code **************************************************************/ + +static const char * +check_code_str(PyUnicodeObject *text) +{ + assert(text != NULL); + if (PyUnicode_GET_LENGTH(text) == 0) { + return "too short"; + } + + // XXX Verify that it parses? + + return NULL; +} + +#ifndef NDEBUG +static int +code_has_args(PyCodeObject *code) +{ + assert(code != NULL); + return (code->co_argcount > 0 + || code->co_posonlyargcount > 0 + || code->co_kwonlyargcount > 0 + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)); +} +#endif + +#define RUN_TEXT 1 +#define RUN_CODE 2 + +static const char * +get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) +{ + const char *codestr = NULL; + Py_ssize_t len = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + + if (PyUnicode_Check(arg)) { + assert(PyUnicode_Check(arg) + && (check_code_str((PyUnicodeObject *)arg) == NULL)); + codestr = PyUnicode_AsUTF8AndSize(arg, &len); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + flags = RUN_TEXT; + } + else { + assert(PyCode_Check(arg)); + assert(_PyCode_VerifyStateless( + PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0); + assert(!code_has_args((PyCodeObject *)arg)); + flags = RUN_CODE; + + // Serialize the code object. + bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); + if (bytes_obj == NULL) { + return NULL; + } + codestr = PyBytes_AS_STRING(bytes_obj); + len = PyBytes_GET_SIZE(bytes_obj); + } + + *flags_p = flags; + *bytes_p = bytes_obj; + *len_p = len; + return codestr; +} + + /* interpreter-specific code ************************************************/ static int @@ -423,14 +499,22 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) static int -_run_script(_PyXIData_t *script, PyObject *ns) +_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { - PyObject *code = _PyXIData_NewObject(script); - if (code == NULL) { - return -1; + PyObject *result = NULL; + if (flags & RUN_TEXT) { + result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + } + else if (flags & RUN_CODE) { + PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); + if (code != NULL) { + result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + } + } + else { + Py_UNREACHABLE(); } - PyObject *result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); if (result == NULL) { return -1; } @@ -439,11 +523,12 @@ _run_script(_PyXIData_t *script, PyObject *ns) } static int -_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, - _PyXIData_t *script, PyObject *shareables, +_run_in_interpreter(PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, + PyObject *shareables, int flags, PyObject **p_excinfo) { - assert(!_PyErr_Occurred(tstate)); + assert(!PyErr_Occurred()); _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { return -1; @@ -451,7 +536,7 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables) < 0) { - if (_PyErr_Occurred(tstate)) { + if (PyErr_Occurred()) { // If an error occured at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); @@ -473,7 +558,7 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, if (mainns == NULL) { goto finally; } - res = _run_script(script, mainns); + res = _run_script(mainns, codestr, codestrlen, flags); finally: // Clean up and switch back. @@ -867,23 +952,104 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static void -unwrap_not_shareable(PyThreadState *tstate) +static PyUnicodeObject * +convert_script_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, + const char *expected) { - PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); - if (!_PyErr_ExceptionMatches(tstate, exctype)) { - return; + PyUnicodeObject *str = NULL; + if (PyUnicode_CheckExact(arg)) { + str = (PyUnicodeObject *)Py_NewRef(arg); } - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *cause = PyException_GetCause(exc); - if (cause != NULL) { - Py_DECREF(exc); - exc = cause; + else if (PyUnicode_Check(arg)) { + // XXX str = PyUnicode_FromObject(arg); + str = (PyUnicodeObject *)Py_NewRef(arg); } else { - assert(PyException_GetContext(exc) == NULL); + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; } + + const char *err = check_code_str(str); + if (err != NULL) { + Py_DECREF(str); + _PyErr_Format(tstate, PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); + return NULL; + } + + return str; +} + +static PyCodeObject * +convert_code_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + PyObject *cause; + PyCodeObject *code = NULL; + if (PyFunction_Check(arg)) { + // For now we allow globals, so we can't use + // _PyFunction_VerifyStateless(). + PyObject *codeobj = PyFunction_GetCode(arg); + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) { + goto chained; + } + code = (PyCodeObject *)Py_NewRef(codeobj); + } + else if (PyCode_Check(arg)) { + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) { + goto chained; + } + code = (PyCodeObject *)Py_NewRef(arg); + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + return code; + +chained: + cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + _PyArg_BadArgument(fname, displayname, expected, arg); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); _PyErr_SetRaisedException(tstate, exc); + return NULL; +} + +static int +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) +{ + if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { + PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); + return -1; + } + + // Extract code. + Py_ssize_t codestrlen = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + const char *codestr = get_code_str(code_arg, + &codestrlen, &bytes_obj, &flags); + if (codestr == NULL) { + return -1; + } + + // Run the code in the interpreter. + int res = _run_in_interpreter(interp, codestr, codestrlen, + shared_arg, flags, p_excinfo); + Py_XDECREF(bytes_obj); + if (res < 0) { + return -1; + } + + return 0; } static PyObject * @@ -896,9 +1062,8 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &code, &PyDict_Type, &shared, - &restricted)) + "OO|O$p:" FUNCNAME, kwlist, + &id, &code, &shared, &restricted)) { return NULL; } @@ -910,17 +1075,22 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // We don't need the script to be "pure", which means it can use - // global variables. They will be resolved against __main__. - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + const char *expected = "a string, a function, or a code object"; + if (PyUnicode_Check(code)) { + code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME, + "argument 2", expected); + } + else { + code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME, + "argument 2", expected); + } + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, code, shared, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -956,9 +1126,8 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O!$p:" FUNCNAME, kwlist, - &id, &script, &PyDict_Type, &shared, - &restricted)) + "OU|O$p:" FUNCNAME, kwlist, + &id, &script, &shared, &restricted)) { return NULL; } @@ -970,20 +1139,15 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (PyFunction_Check(script) || PyCode_Check(script)) { - _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); - return NULL; - } - - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { - unwrap_not_shareable(tstate); + script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME, + "argument 2", "a string"); + if (script == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, script, shared, &excinfo); + Py_DECREF(script); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1009,9 +1173,8 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &func, &PyDict_Type, &shared, - &restricted)) + "OO|O$p:" FUNCNAME, kwlist, + &id, &func, &shared, &restricted)) { return NULL; } @@ -1023,29 +1186,16 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // We don't worry about checking globals. They will be resolved - // against __main__. - PyObject *code; - if (PyFunction_Check(func)) { - code = PyFunction_GET_CODE(func); - } - else if (PyCode_Check(func)) { - code = func; - } - else { - _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); - return NULL; - } - - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME, + "argument 2", + "a function or a code object"); + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1098,15 +1248,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - _PyXIData_t xidata = {0}; - if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) { - unwrap_not_shareable(tstate); + PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME, + "argument 2", "a function"); + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, code, NULL, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo;