diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 462b65bc7afba4..95ef72469b18ef 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -231,6 +231,42 @@ Heap elements can be tuples. This is useful for assigning comparison values (1, 'write spec') +Other Applications +------------------ + +`Medians `_ are a measure of +central tendency for a set of numbers. In distributions skewed by +outliers, the median provides a more stable estimate than an average +(arithmetic mean). A running median is an `online algorithm +`_ that updates +continuously as new data arrives. + +A running median can be efficiently implemented by balancing two heaps, +a max-heap for values at or below the midpoint and a min-heap for values +above the midpoint. When the two heaps have the same size, the new +median is the average of the tops of the two heaps; otherwise, the +median is at the top of the larger heap:: + + def running_median(iterable): + "Yields the cumulative median of values seen so far." + + lo = [] # max-heap + hi = [] # min-heap (same size as or one smaller than lo) + + for x in iterable: + if len(lo) == len(hi): + heappush_max(lo, heappushpop(hi, x)) + yield lo[0] + else: + heappush(hi, heappushpop_max(lo, x)) + yield (lo[0] + hi[0]) / 2 + +For example:: + + >>> list(running_median([5.0, 9.0, 4.0, 12.0, 8.0, 9.0])) + [5.0, 7.0, 5.0, 7.0, 8.0, 8.5] + + Priority Queue Implementation Notes ----------------------------------- diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 7eb55bd63ddb73..f404273c3ae5c1 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -460,6 +460,8 @@ def _pipe_closed(self, fut): class _ProactorDatagramTransport(_ProactorBasePipeTransport, transports.DatagramTransport): max_size = 256 * 1024 + _header_size = 8 + def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address @@ -499,7 +501,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size if self._write_fut is None: # No current write operations are active, kick one off @@ -526,7 +528,7 @@ def _loop_writing(self, fut=None): return data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size if self._address is not None: self._write_fut = self._loop._proactor.send(self._sock, data) diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 6ad84044adf146..3505d4bb6bd1f7 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1212,6 +1212,7 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): @@ -1285,13 +1286,13 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size try: if self._extra['peername']: self._sock.send(data) @@ -1299,7 +1300,7 @@ def _sendto_ready(self): self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) diff --git a/Lib/heapq.py b/Lib/heapq.py index 6ceb211f1ca2ae..17f62dd2d5839b 100644 --- a/Lib/heapq.py +++ b/Lib/heapq.py @@ -126,8 +126,9 @@ From all times, sorting has always been a Great Art! :-) """ -__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'merge', - 'nlargest', 'nsmallest', 'heappushpop'] +__all__ = ['heappush', 'heappop', 'heapify', 'heapreplace', 'heappushpop', + 'heappush_max', 'heappop_max', 'heapify_max', 'heapreplace_max', + 'heappushpop_max', 'nlargest', 'nsmallest', 'merge'] def heappush(heap, item): """Push item onto heap, maintaining the heap invariant.""" diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index b25daaface0807..edfad5e11db35e 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -566,6 +566,8 @@ def test_sendto(self): self.assertTrue(self.proactor.sendto.called) self.proactor.sendto.assert_called_with( self.sock, data, addr=('0.0.0.0', 1234)) + self.assertFalse(transport._buffer) + self.assertEqual(0, transport._buffer_size) def test_sendto_bytearray(self): data = bytearray(b'data') diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 7b6d1bce5e460f..9d77e7e5889d61 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1497,6 +1497,47 @@ def test_sendto_closing(self): transport.sendto(b'data', (1,)) self.assertEqual(transport._conn_lost, 2) + def test_sendto_sendto_ready(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now let the socket send the buffer + self.sock.sendto.side_effect = None + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertFalse(self.loop.writers) + self.assertFalse(transport._buffer) + self.assertEqual(transport._buffer_size, 0) + + def test_sendto_sendto_ready_blocked(self): + data = b'data' + + # First queue up the buffer by having the socket blocked + self.sock.sendto.side_effect = BlockingIOError + transport = self.datagram_transport() + transport.sendto(data, ('0.0.0.0', 12345)) + self.loop.assert_writer(7, transport._sendto_ready) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + + # Now try to send the buffer, it will be added to buffer again if it fails + transport._sendto_ready() + self.assertTrue(self.sock.sendto.called) + self.assertEqual( + self.sock.sendto.call_args[0], (data, ('0.0.0.0', 12345))) + self.assertTrue(self.loop.writers) + self.assertEqual(1, len(transport._buffer)) + self.assertEqual(transport._buffer_size, len(data) + transport._header_size) + def test_sendto_ready(self): data = b'data' self.sock.sendto.return_value = len(data) diff --git a/Misc/NEWS.d/next/Library/2025-06-16-12-37-02.gh-issue-135444.An2eeA.rst b/Misc/NEWS.d/next/Library/2025-06-16-12-37-02.gh-issue-135444.An2eeA.rst new file mode 100644 index 00000000000000..e1182f56eb33ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-16-12-37-02.gh-issue-135444.An2eeA.rst @@ -0,0 +1,2 @@ +Fix :meth:`asyncio.DatagramTransport.sendto` to account for datagram header size when +data cannot be sent. diff --git a/Misc/NEWS.d/next/Library/2025-07-30-17-42-36.gh-issue-137239.qSpj32.rst b/Misc/NEWS.d/next/Library/2025-07-30-17-42-36.gh-issue-137239.qSpj32.rst new file mode 100644 index 00000000000000..3be583ee937f8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-30-17-42-36.gh-issue-137239.qSpj32.rst @@ -0,0 +1 @@ +:mod:`heapq`: Update :data:`!heapq.__all__` with ``*_max`` functions. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-07-30-10-28-35.gh-issue-137243.NkdUqH.rst b/Misc/NEWS.d/next/Tools-Demos/2025-07-30-10-28-35.gh-issue-137243.NkdUqH.rst new file mode 100644 index 00000000000000..c9c6c2ca287efd --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-07-30-10-28-35.gh-issue-137243.NkdUqH.rst @@ -0,0 +1,2 @@ +Have Tools/wasm/wasi detect a WASI SDK install in /opt when it was directly +extracted from a release tarball. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-07-30-11-15-47.gh-issue-137248.8IxwY3.rst b/Misc/NEWS.d/next/Tools-Demos/2025-07-30-11-15-47.gh-issue-137248.8IxwY3.rst new file mode 100644 index 00000000000000..311ade0c8f3040 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-07-30-11-15-47.gh-issue-137248.8IxwY3.rst @@ -0,0 +1,2 @@ +Add a ``--logdir`` option to ``Tools/wasm/wasi`` for specifying where to +write log files. diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py index f3c97ff3fd11a0..fdd93d5c0aa4af 100644 --- a/Tools/wasm/wasi/__main__.py +++ b/Tools/wasm/wasi/__main__.py @@ -26,6 +26,8 @@ LOCAL_SETUP_MARKER = ("# Generated by Tools/wasm/wasi .\n" "# Required to statically build extension modules.").encode("utf-8") +WASI_SDK_VERSION = 24 + WASMTIME_VAR_NAME = "WASMTIME" WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" @@ -91,11 +93,17 @@ def wrapper(context): return decorator -def call(command, *, quiet, **kwargs): +def call(command, *, context=None, quiet=False, logdir=None, **kwargs): """Execute a command. If 'quiet' is true, then redirect stdout and stderr to a temporary file. """ + if context is not None: + quiet = context.quiet + logdir = context.logdir + elif quiet and logdir is None: + raise ValueError("When quiet is True, logdir must be specified") + print("❯", " ".join(map(str, command))) if not quiet: stdout = None @@ -103,6 +111,7 @@ def call(command, *, quiet, **kwargs): else: stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", delete=False, + dir=logdir, prefix="cpython-wasi-", suffix=".log") stderr = subprocess.STDOUT @@ -154,14 +163,14 @@ def configure_build_python(context, working_dir): if context.args: configure.extend(context.args) - call(configure, quiet=context.quiet) + call(configure, context=context) @subdir(BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) + context=context) binary = build_python_path() cmd = [binary, "-c", @@ -173,10 +182,22 @@ def make_build_python(context, working_dir): def find_wasi_sdk(): - """Find the path to wasi-sdk.""" + """Find the path to the WASI SDK.""" if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): return pathlib.Path(wasi_sdk_path) - elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): + + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [path for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + if path.is_dir()] + if len(potential_sdks) == 1: + return potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): return default_path @@ -261,7 +282,7 @@ def configure_wasi_python(context, working_dir): configure.extend(context.args) call(configure, env=updated_env(env_additions | wasi_sdk_env(context)), - quiet=context.quiet) + context=context) python_wasm = working_dir / "python.wasm" exec_script = working_dir / "python.sh" @@ -277,7 +298,7 @@ def make_wasi_python(context, working_dir): """Run `make` for the WASI/host build.""" call(["make", "--jobs", str(cpu_count()), "all"], env=updated_env(), - quiet=context.quiet) + context=context) exec_script = working_dir / "python.sh" call([exec_script, "--version"], quiet=False) @@ -306,6 +327,8 @@ def clean_contents(context): def main(): + default_host_triple = "wasm32-wasip1" + default_wasi_sdk = find_wasi_sdk() default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " # Make sure the stack size will work for a pydebug # build. @@ -317,6 +340,7 @@ def main(): "--dir {HOST_DIR}::{GUEST_DIR} " # Set PYTHONPATH to the sysconfig data. "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") + default_logdir = pathlib.Path(tempfile.gettempdir()) parser = argparse.ArgumentParser() subcommands = parser.add_subparsers(dest="subcommand") @@ -339,6 +363,9 @@ def main(): subcommand.add_argument("--quiet", action="store_true", default=False, dest="quiet", help="Redirect output from subprocesses to a log file") + subcommand.add_argument("--logdir", type=pathlib.Path, default=default_logdir, + help="Directory to store log files; " + f"defaults to {default_logdir}") for subcommand in configure_build, configure_host: subcommand.add_argument("--clean", action="store_true", default=False, dest="clean", @@ -349,17 +376,17 @@ def main(): for subcommand in build, configure_host: subcommand.add_argument("--wasi-sdk", type=pathlib.Path, dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}") subcommand.add_argument("--host-runner", action="store", default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host " - "(default designed for wasmtime 14 or newer: " - f"`{default_host_runner}`)") + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`") for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", - help="The target triple for the WASI host build") + subcommand.add_argument("--host-triple", action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}") context = parser.parse_args() context.init_dir = pathlib.Path().absolute()