diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 61fa49f8681cce..df1bb0ce370919 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -672,6 +672,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index c5ca599bb04a6f..9505bcfa05af11 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -45,7 +45,7 @@ Pending removal in Python 3.15 * :mod:`pathlib`: - * :meth:`.PurePath.is_reserved` + * :meth:`!.PurePath.is_reserved` has been deprecated since Python 3.13. Use :func:`os.path.isreserved` to detect reserved paths on Windows. diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f189f6b8fa8953..a08f713ab56ba3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -434,12 +434,18 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Empty lines are treated as empty strings (``''``), which are allowed as values but + not as arguments. Empty lines that are read as arguments will result in an + "unrecognized arguments" error. + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 846cece3761858..09f596101b4d1e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -232,8 +232,24 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_int` | :c:expr:`int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_int8` | :c:type:`int8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int16` | :c:type:`int16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int32` | :c:type:`int32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int64` | :c:type:`int64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_uint` | :c:expr:`unsigned int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint8` | :c:type:`uint8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint16` | :c:type:`uint16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint32` | :c:type:`uint32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint64` | :c:type:`uint64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_long` | :c:expr:`long` | int | +----------------------+------------------------------------------+----------------------------+ | :class:`c_ulong` | :c:expr:`unsigned long` | int | @@ -2524,7 +2540,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2599,7 +2615,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 12e61bc36f5db0..ee654b7a83e203 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -53,7 +53,7 @@ a :class:`!str` filename, and vice-versa. Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following -functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. .. function:: fnmatch(name, pat) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index 392b6d40e861fb..8796056b4b8722 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. + with value equal to ``numerator/denominator`` where the denominator is positive. + If *denominator* is ``0``, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index e5bdfbb144b65a..9e887d8e65741b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -240,7 +240,16 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_reserved - ``True`` if the address is otherwise IETF reserved. + ``True`` if the address is noted as reserved by the IETF. + For IPv4, this is only ``240.0.0.0/4``, the ``Reserved`` address block. + For IPv6, this is all addresses `allocated `__ as + ``Reserved by IETF`` for future use. + + .. note:: For IPv4, ``is_reserved`` is not related to the address block value of the + ``Reserved-by-Protocol`` column in iana-ipv4-special-registry_. + + .. caution:: For IPv6, ``fec0::/10`` a former Site-Local scoped address prefix is + currently excluded from that list (see :attr:`~IPv6Address.is_site_local` & :rfc:`3879`). .. attribute:: is_loopback @@ -261,6 +270,7 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. _iana-ipv6-address-space: https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml .. method:: IPv4Address.__format__(fmt) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 546876bd925db0..c80f78e614818e 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -936,8 +936,13 @@ For an example of the usage of queues for interprocess communication see .. method:: close() - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`~Queue.get`, :meth:`~Queue.put` and :meth:`~Queue.empty` + methods must no longer be called. + + The background thread will quit once it has flushed all buffered data to the pipe. This is called automatically when the queue is garbage collected. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 47986a2d9602ee..ebf5756146df92 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -542,20 +542,6 @@ Pure paths provide the following methods and properties: Passing additional arguments is deprecated; if supplied, they are joined with *other*. -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - .. versionchanged:: 3.13 - Windows path names that contain a colon, or end with a dot or a space, - are considered reserved. UNC paths may be reserved. - - .. deprecated-removed:: 3.13 3.15 - This method is deprecated; use :func:`os.path.isreserved` to detect - reserved paths on Windows. - .. method:: PurePath.joinpath(*pathsegments) Calling this method is equivalent to combining the path with each of diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 58bd111b5cc374..1e716715fd9bed 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -829,10 +829,13 @@ The following attribute and methods should only be used by classes derived from errors. It will be called automatically by the :class:`OpenerDirector` getting the error, and should not normally be called in other circumstances. - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. + :class:`OpenerDirector` will call this method with five positional arguments: + + 1. a :class:`Request` object, + #. a file-like object with the HTTP error body, + #. the three-digit code of the error, as a string, + #. the user-visible explanation of the code, as as string, and + #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of :func:`urlopen`. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e5fe3c72b1b26e..2cda9587975ddc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -802,6 +802,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). + To improve ASan detection capabilities you may also want to combine this + with :option:`--without-pymalloc` to disable the specialized small-object + allocator whose allocations are not tracked by ASan. .. versionadded:: 3.6 diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 0a3b3b30e016da..126329b538db31 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -1917,7 +1917,7 @@ New Deprecations * :mod:`pathlib`: - * Deprecate :meth:`.PurePath.is_reserved`, + * Deprecate :meth:`!.PurePath.is_reserved`, to be removed in Python 3.15. Use :func:`os.path.isreserved` to detect reserved paths on Windows. (Contributed by Barney Gale in :gh:`88569`.) diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index fe3d45b83a512e..0f65317633ba70 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -82,7 +82,7 @@ running Python processes without requiring code modification or process restart. Unlike deterministic profilers (:mod:`cProfile` and :mod:`profile`) that instrument every function call, the sampling profiler periodically captures stack traces from running processes. This approach provides virtually zero overhead while achieving -sampling rates of **up to 200,000 Hz**, making it the fastest sampling profiler +sampling rates of **up to 1,000,000 Hz**, making it the fastest sampling profiler available for Python (at the time of its contribution) and ideal for debugging performance issues in production environments. @@ -373,6 +373,14 @@ http.server (Contributed by Bénédikt Tran in :gh:`133810`.) +pathlib +------- + +* Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. + Use :func:`os.path.isreserved` to detect reserved paths on Windows. + (Contributed by Nikita Sobolev in :gh:`133875`.) + + platform -------- @@ -482,6 +490,7 @@ Porting to Python 3.15 The |pythoncapi_compat_project| can be used to get most of these new functions on Python 3.14 and older. + Deprecated C APIs ----------------- diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index e04fbdc6c8a5c4..fd788c8429e15b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -241,14 +241,14 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return s in keyword_first_sets_match return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) | TI(T.OP, string="(" | "*" | "-" | "[" | "{") ): return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NAME, string=s) ): diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 86d29df0639184..83ea623dce6281 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1265,7 +1265,7 @@ def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): doc = getattr(defined_fields.get(slot), 'doc', None) if doc is not None: seen_docs = True - slots.update({slot: doc}) + slots[slot] = doc # We only return dict if there's at least one doc member, # otherwise we return tuple, which is the old default format. diff --git a/Lib/enum.py b/Lib/enum.py index 538b9cc8e96fc1..c00ae85d2f8efe 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1991,7 +1991,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 05633ac21a259c..c4d0ca81e7034a 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -47,12 +47,8 @@ def cleanup_noop(name): # absence of POSIX named semaphores. In that case, no named semaphores were # ever opened, so no cleanup would be necessary. if hasattr(_multiprocessing, 'sem_unlink'): - _CLEANUP_FUNCS.update({ - 'semaphore': _multiprocessing.sem_unlink, - }) - _CLEANUP_FUNCS.update({ - 'shared_memory': _posixshmem.shm_unlink, - }) + _CLEANUP_FUNCS['semaphore'] = _multiprocessing.sem_unlink + _CLEANUP_FUNCS['shared_memory'] = _posixshmem.shm_unlink class ReentrantCallError(RuntimeError): diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 2dc1f7f7126063..fd073445e89b62 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -519,18 +519,6 @@ def is_absolute(self): return False return self.parser.isabs(self) - def is_reserved(self): - """Return True if the path contains one of the special names reserved - by the system, if any.""" - import warnings - msg = ("pathlib.PurePath.is_reserved() is deprecated and scheduled " - "for removal in Python 3.15. Use os.path.isreserved() to " - "detect reserved paths on Windows.") - warnings._deprecated("pathlib.PurePath.is_reserved", msg, remove=(3, 15)) - if self.parser is ntpath: - return self.parser.isreserved(self) - return False - def as_uri(self): """Return the path as a URI.""" import warnings diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py new file mode 100644 index 00000000000000..272d9a10ceb950 --- /dev/null +++ b/Lib/test/test_capi/test_emscripten.py @@ -0,0 +1,25 @@ +import unittest +from test.support import is_emscripten + +if not is_emscripten: + raise unittest.SkipTest("Emscripten-only test") + +from _testinternalcapi import emscripten_set_up_async_input_device +from pathlib import Path + + +class EmscriptenAsyncInputDeviceTest(unittest.TestCase): + def test_emscripten_async_input_device(self): + jspi_supported = emscripten_set_up_async_input_device() + p = Path("/dev/blah") + self.addCleanup(p.unlink) + if not jspi_supported: + with open(p, "r") as f: + self.assertRaises(OSError, f.readline) + return + + with open(p, "r") as f: + for _ in range(10): + self.assertEqual(f.readline().strip(), "ab") + self.assertEqual(f.readline().strip(), "fi") + self.assertEqual(f.readline().strip(), "xy") diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index b2e2cdb3338beb..16d30e3ca2d17d 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -539,12 +539,6 @@ def test_with_stem_empty(self): self.assertRaises(ValueError, P('a/b').with_stem, '') self.assertRaises(ValueError, P('a/b').with_stem, '.') - def test_is_reserved_deprecated(self): - P = self.cls - p = P('a/b') - with self.assertWarns(DeprecationWarning): - p.is_reserved() - def test_full_match_case_sensitive(self): P = self.cls self.assertFalse(P('A.py').full_match('a.PY', case_sensitive=True)) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 1f655264f1c00a..9a02dff7387563 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -375,7 +375,8 @@ def funct(case: str = sys.platform) -> None: ) match case: case "emscripten": print("on the web") - case "ios" | "android": print("on the phone") + case "ios" | "android": + print("on the phone") case _: print('arms around', match.group(1)) """ ) @@ -393,7 +394,8 @@ def funct(case: str = sys.platform) -> None: {o}){z} {K}match{z} case{o}:{z} {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} - {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} """ ) @@ -402,14 +404,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 2**7 + 2**8) - self.assertEqual(reader.cxy, (0, 14)) + self.assertEqual(reader.pos, 396) + self.assertEqual(reader.cxy, (0, 15)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 13, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 14, code_to_events("async "), ) reader, _ = handle_all_events(more_events) diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py index 4f9a8815b6184b..8dcead5e31930e 100644 --- a/Lib/tkinter/scrolledtext.py +++ b/Lib/tkinter/scrolledtext.py @@ -23,7 +23,7 @@ def __init__(self, master=None, **kw): self.vbar = Scrollbar(self.frame) self.vbar.pack(side=RIGHT, fill=Y) - kw.update({'yscrollcommand': self.vbar.set}) + kw['yscrollcommand'] = self.vbar.set Text.__init__(self, self.frame, **kw) self.pack(side=LEFT, fill=BOTH, expand=True) self.vbar['command'] = self.yview diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 8130c739af2fe8..3e6871157d0929 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 0b49c2a78771d2..2c8e349d3c8bfb 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -2741,7 +2741,7 @@ Fix deprecation of :data:`ssl.OP_NO_TLSv1_3` .. nonce: TMWh1i .. section: Library -:meth:`pathlib.PureWindowsPath.is_reserved` now identifies a greater range +:meth:`!pathlib.PureWindowsPath.is_reserved` now identifies a greater range of reserved filenames, including those with trailing spaces or colons. .. diff --git a/Misc/NEWS.d/3.13.0a4.rst b/Misc/NEWS.d/3.13.0a4.rst index 1b971113173e0a..8afbe1b77f7cb8 100644 --- a/Misc/NEWS.d/3.13.0a4.rst +++ b/Misc/NEWS.d/3.13.0a4.rst @@ -1096,7 +1096,7 @@ Also changed its name and daemonic status, it can be now joined. Add :func:`os.path.isreserved`, which identifies reserved pathnames such as "NUL", "AUX" and "CON". This function is only available on Windows. -Deprecate :meth:`pathlib.PurePath.is_reserved`. +Deprecate :meth:`!pathlib.PurePath.is_reserved`. .. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst new file mode 100644 index 00000000000000..5c0813b1a0abda --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst @@ -0,0 +1 @@ +Fix PyREPL syntax highlightning on match cases after multi-line case. Contributed by Olga Matoula. diff --git a/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst b/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst new file mode 100644 index 00000000000000..b4a2b0336370bc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-11-11-39-05.gh-issue-133875.pUar3l.rst @@ -0,0 +1,2 @@ +Removed deprecated :meth:`!pathlib.PurePath.is_reserved`. Use +:func:`os.path.isreserved` to detect reserved paths on Windows. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 533e7dd3a7ec00..243c7346576fc6 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2345,6 +2345,37 @@ incref_decref_delayed(PyObject *self, PyObject *op) Py_RETURN_NONE; } +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" + +EM_JS(int, emscripten_set_up_async_input_device_js, (void), { + let idx = 0; + const encoder = new TextEncoder(); + const bufs = [ + encoder.encode("ab\n"), + encoder.encode("fi\n"), + encoder.encode("xy\n"), + ]; + function sleep(t) { + return new Promise(res => setTimeout(res, t)); + } + FS.createAsyncInputDevice("/dev", "blah", async () => { + await sleep(5); + return bufs[(idx ++) % 3]; + }); + return !!WebAssembly.promising; +}); + +static PyObject * +emscripten_set_up_async_input_device(PyObject *self, PyObject *Py_UNUSED(ignored)) { + if (emscripten_set_up_async_input_device_js()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +#endif + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2447,6 +2478,9 @@ static PyMethodDef module_functions[] = { {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF +#ifdef __EMSCRIPTEN__ + {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index bb80f979420ec1..bd5cc07071f20e 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -37,3 +37,178 @@ EM_JS(int, __syscall_umask_js, (int mask), { int __syscall_umask(int mask) { return __syscall_umask_js(mask); } + +#include +#include +#undef errno + +// Variant of EM_JS that does C preprocessor substitution on the body +#define EM_JS_MACROS(ret, func_name, args, body...) \ + EM_JS(ret, func_name, args, body) + +EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { + // Define FS.createAsyncInputDevice(), This is quite similar to + // FS.createDevice() defined here: + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 + // but instead of returning one byte at a time, the input() function should + // return a Uint8Array. This makes the handler code simpler, the + // `createAsyncInputDevice` simpler, and everything faster. + FS.createAsyncInputDevice = function(parent, name, input) { + parent = typeof parent == 'string' ? parent : FS.getPath(parent); + var path = PATH.join2(parent, name); + var mode = FS_getMode(true, false); + FS.createDevice.major ||= 64; + var dev = FS.makedev(FS.createDevice.major++, 0); + async function getDataBuf() { + var buf; + try { + buf = await input(); + } catch (e) { + throw new FS.ErrnoError(EIO); + } + if (!buf?.byteLength) { + throw new FS.ErrnoError(EAGAIN); + } + ops._dataBuf = buf; + } + + var ops = { + _dataBuf: new Uint8Array(0), + open(stream) { + stream.seekable = false; + }, + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + buffer = buffer.subarray(offset, offset + length); + if (!ops._dataBuf.byteLength) { + await getDataBuf(); + } + var toRead = Math.min(ops._dataBuf.byteLength, buffer.byteLength); + buffer.subarray(0, toRead).set(ops._dataBuf); + buffer = buffer.subarray(toRead); + ops._dataBuf = ops._dataBuf.subarray(toRead); + if (toRead) { + stream.node.atime = Date.now(); + } + return toRead; + }, + }; + FS.registerDevice(dev, ops); + return FS.mkdev(path, mode, dev); + }; + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + const origResolveGlobalSymbol = resolveGlobalSymbol; + if (!Module.onExit && process?.exit) { + Module.onExit = (code) => process.exit(code); + } + // * wrap the main symbol with WebAssembly.promising, + // * call exit_with_live_runtime() to prevent emscripten from shutting down + // the runtime before the promise resolves, + // * call onExit / process.exit ourselves, since exit_with_live_runtime() + // prevented Emscripten from calling it normally. + resolveGlobalSymbol = function (name, direct = false) { + const orig = origResolveGlobalSymbol(name, direct); + if (name === "main") { + const main = WebAssembly.promising(orig.sym); + orig.sym = (...args) => { + (async () => { + const ret = await main(...args); + process?.exit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + } + return orig; + }; +}) + +__attribute__((constructor)) void _emscripten_promising_main(void) { + _emscripten_promising_main_js(); +} + + +#define IOVEC_T_BUF_OFFSET 0 +#define IOVEC_T_BUF_LEN_OFFSET 4 +#define IOVEC_T_SIZE 8 +_Static_assert(offsetof(__wasi_iovec_t, buf) == IOVEC_T_BUF_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(offsetof(__wasi_iovec_t, buf_len) == IOVEC_T_BUF_LEN_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, + "Unexpected __wasi_iovec_t layout"); + +// If the stream has a readAsync handler, read to buffer defined in iovs, write +// number of bytes read to *nread, and return a promise that resolves to the +// errno. Otherwise, return null. +EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( + __wasi_fd_t fd, + const __wasi_iovec_t *iovs, + size_t iovcnt, + __wasi_size_t *nread +), { + var stream = SYSCALLS.getStreamFromFD(fd); + if (!WebAssembly.promising) { + return null; + } + if (!stream.stream_ops.readAsync) { + // Not an async device. Fall back to __wasi_fd_read_orig(). + return null; + } + return (async () => { + // This is the same as libwasi.js fd_read() and doReadv() except we use + // readAsync and we await it. + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L331 + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L197 + try { + var ret = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[(iovs + IOVEC_T_BUF_OFFSET)/4]; + var len = HEAP32[(iovs + IOVEC_T_BUF_LEN_OFFSET)/4]; + iovs += IOVEC_T_SIZE; + var curr = await stream.stream_ops.readAsync(stream, HEAP8, ptr, len); + if (curr < 0) return -1; + ret += curr; + if (curr < len) break; // nothing more to read + } + HEAP32[nread/4] = ret; + return 0; + } catch (e) { + if (e.name !== 'ErrnoError') { + throw e; + } + return e.errno; + } + })(); +}; +); + +// Bind original fd_read syscall to __wasi_fd_read_orig(). +__wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("fd_read"), __warn_unused_result__)); + +// Take a promise that resolves to __wasi_errno_t and suspend until it resolves, +// get the output. +EM_JS(__wasi_errno_t, __block_for_errno, (__externref_t p), { + return p; +} +if (WebAssembly.Suspending) { + __block_for_errno = new WebAssembly.Suspending(__block_for_errno); +} +) + +// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_errno +// to get the result. +__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) { + __externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread); + if (__builtin_wasm_ref_is_null_extern(p)) { + return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); + } + __wasi_errno_t res = __block_for_errno(p); + return res; +} diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index c0d58aeaadd2cf..dc8422420a564c 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -263,10 +263,20 @@ def configure_emscripten_python(context, working_dir): REALPATH=abs_path fi + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) )