Catch up with main

Author: brandtbucher

Doc/conf.py

@@ -28,6 +28,7 @@
     'changes',
     'glossary_search',
     'grammar_snippet',
+    'implementation_detail',
     'lexers',
     'misc_news',
     'pydoc_topics',

Doc/library/imaplib.rst

@@ -10,6 +10,7 @@
 .. changes for IMAP4_SSL by Tino Lange <[email protected]>, March 2002
 .. changes for IMAP4_stream by Piers Lauder <[email protected]>,
    November 2002
+.. changes for IMAP4 IDLE by Forest <[email protected]>, August 2024
 
 **Source code:** :source:`Lib/imaplib.py`
 
@@ -187,7 +188,7 @@ However, the *password* argument to the ``LOGIN`` command is always quoted. If
 you want to avoid having an argument string quoted (eg: the *flags* argument to
 ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``).
 
-Each command returns a tuple: ``(type, [data, ...])`` where *type* is usually
+Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually
 ``'OK'`` or ``'NO'``, and *data* is either the text from the command response,
 or mandated results from the command. Each *data* is either a ``bytes``, or a
 tuple. If a tuple, then the first part is the header of the response, and the
@@ -307,6 +308,93 @@ An :class:`IMAP4` instance has the following methods:
    of the IMAP4 QUOTA extension defined in rfc2087.
 
 
+.. method:: IMAP4.idle(duration=None)
+
+   Return an :class:`!Idler`: an iterable context manager implementing the
+   IMAP4 ``IDLE`` command as defined in :rfc:`2177`.
+
+   The returned object sends the ``IDLE`` command when activated by the
+   :keyword:`with` statement, produces IMAP untagged responses via the
+   :term:`iterator` protocol, and sends ``DONE`` upon context exit.
+
+   All untagged responses that arrive after sending the ``IDLE`` command
+   (including any that arrive before the server acknowledges the command) will
+   be available via iteration. Any leftover responses (those not iterated in
+   the :keyword:`with` context) can be retrieved in the usual way after
+   ``IDLE`` ends, using :meth:`IMAP4.response`.
+
+   Responses are represented as ``(type, [data, ...])`` tuples, as described
+   in :ref:`IMAP4 Objects <imap4-objects>`.
+
+   The *duration* argument sets a maximum duration (in seconds) to keep idling,
+   after which any ongoing iteration will stop. It can be an :class:`int` or
+   :class:`float`, or ``None`` for no time limit.
+   Callers wishing to avoid inactivity timeouts on servers that impose them
+   should keep this at most 29 minutes (1740 seconds).
+   Requires a socket connection; *duration* must be ``None`` on
+   :class:`IMAP4_stream` connections.
+
+   .. code-block:: pycon
+
+      >>> with M.idle(duration=29 * 60) as idler:
+      ...     for typ, data in idler:
+      ...         print(typ, data)
+      ...
+      EXISTS [b'1']
+      RECENT [b'1']
+
+
+   .. method:: Idler.burst(interval=0.1)
+
+      Yield a burst of responses no more than *interval* seconds apart
+      (expressed as an :class:`int` or :class:`float`).
+
+      This :term:`generator` is an alternative to iterating one response at a
+      time, intended to aid in efficient batch processing. It retrieves the
+      next response along with any immediately available subsequent responses.
+      (For example, a rapid series of ``EXPUNGE`` responses after a bulk
+      delete.)
+
+      Requires a socket connection; does not work on :class:`IMAP4_stream`
+      connections.
+
+      .. code-block:: pycon
+
+         >>> with M.idle() as idler:
+         ...     # get a response and any others following by < 0.1 seconds
+         ...     batch = list(idler.burst())
+         ...     print(f'processing {len(batch)} responses...')
+         ...     print(batch)
+         ...
+         processing 3 responses...
+         [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])]
+
+      .. tip::
+
+         The ``IDLE`` context's maximum duration, as passed to
+         :meth:`IMAP4.idle`, is respected when waiting for the first response
+         in a burst. Therefore, an expired :class:`!Idler` will cause this
+         generator to return immediately without producing anything. Callers
+         should consider this if using it in a loop.
+
+
+   .. note::
+
+      The iterator returned by :meth:`IMAP4.idle` is usable only within a
+      :keyword:`with` statement. Before or after that context, unsolicited
+      responses are collected internally whenever a command finishes, and can
+      be retrieved with :meth:`IMAP4.response`.
+
+   .. note::
+
+      The :class:`!Idler` class name and structure are internal interfaces,
+      subject to change. Calling code can rely on its context management,
+      iteration, and public method to remain stable, but should not subclass,
+      instantiate, compare, or otherwise directly reference the class.
+
+   .. versionadded:: next
+
+
 .. method:: IMAP4.list([directory[, pattern]])
 
    List mailbox names in *directory* matching *pattern*.  *directory* defaults to

Doc/library/itertools.rst

@@ -838,10 +838,10 @@ and :term:`generators <generator>` which incur interpreter overhead.
 
 .. testcode::
 
-   from collections import deque
+   from collections import Counter, deque
    from contextlib import suppress
    from functools import reduce
-   from math import sumprod, isqrt
+   from math import comb, prod, sumprod, isqrt
    from operator import itemgetter, getitem, mul, neg
 
    def take(n, iterable):
@@ -1127,6 +1127,12 @@ The following recipes have a more mathematical flavor:
            n -= n // prime
        return n
 
+   def multinomial(*counts):
+       "Number of distinct arrangements of a multiset."
+       # Counter('abracadabra').values() -> 5 2 1 1 2
+       # multinomial(5, 2, 1, 1, 2) → 83160
+       return prod(map(comb, accumulate(counts), counts))
+
 
 .. doctest::
     :hide:
@@ -1730,6 +1736,12 @@ The following recipes have a more mathematical flavor:
     >>> ''.join(it)
     'DEF1'
 
+    >>> multinomial(5, 2, 1, 1, 2)
+    83160
+    >>> word = 'coffee'
+    >>> multinomial(*Counter(word).values()) == len(set(permutations(word)))
+    True
+
 
 .. testcode::
     :hide:

Doc/library/socket.rst

@@ -1668,11 +1668,6 @@ to sockets.
    See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument
    *flags*; it defaults to zero.
 
-   .. note::
-
-      For best match with hardware and network realities, the value of  *bufsize*
-      should be a relatively small power of 2, for example, 4096.
-
    .. versionchanged:: 3.5
       If the system call is interrupted and the signal handler does not raise
       an exception, the method now retries the system call instead of raising

Doc/library/socketserver.rst

@@ -499,11 +499,17 @@ This is the server side::
 
        def handle(self):
            # self.request is the TCP socket connected to the client
-           self.data = self.request.recv(1024).strip()
-           print("Received from {}:".format(self.client_address[0]))
-           print(self.data)
+           pieces = [b'']
+           total = 0
+           while b'\n' not in pieces[-1] and total < 10_000:
+               pieces.append(self.request.recv(2000))
+               total += len(pieces[-1])
+           self.data = b''.join(pieces)
+           print(f"Received from {self.client_address[0]}:")
+           print(self.data.decode("utf-8"))
            # just send back the same data, but upper-cased
            self.request.sendall(self.data.upper())
+           # after we return, the socket will be closed.
 
    if __name__ == "__main__":
        HOST, PORT = "localhost", 9999
@@ -520,20 +526,24 @@ objects that simplify communication by providing the standard file interface)::
    class MyTCPHandler(socketserver.StreamRequestHandler):
 
        def handle(self):
-           # self.rfile is a file-like object created by the handler;
-           # we can now use e.g. readline() instead of raw recv() calls
-           self.data = self.rfile.readline().strip()
-           print("{} wrote:".format(self.client_address[0]))
-           print(self.data)
+           # self.rfile is a file-like object created by the handler.
+           # We can now use e.g. readline() instead of raw recv() calls.
+           # We limit ourselves to 10000 bytes to avoid abuse by the sender.
+           self.data = self.rfile.readline(10000).rstrip()
+           print(f"{self.client_address[0]} wrote:")
+           print(self.data.decode("utf-8"))
            # Likewise, self.wfile is a file-like object used to write back
            # to the client
            self.wfile.write(self.data.upper())
 
 The difference is that the ``readline()`` call in the second handler will call
 ``recv()`` multiple times until it encounters a newline character, while the
-single ``recv()`` call in the first handler will just return what has been
-received so far from the client's ``sendall()`` call (typically all of it, but
-this is not guaranteed by the TCP protocol).
+the first handler had to use a ``recv()`` loop to accumulate data until a
+newline itself.  If it had just used a single ``recv()`` without the loop it
+would just have returned what has been received so far from the client.
+TCP is stream based: data arrives in the order it was sent, but there no
+correlation between client ``send()`` or ``sendall()`` calls and the number
+of ``recv()`` calls on the server required to receive it.
 
 
 This is the client side::
@@ -548,13 +558,14 @@ This is the client side::
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
        # Connect to server and send data
        sock.connect((HOST, PORT))
-       sock.sendall(bytes(data + "\n", "utf-8"))
+       sock.sendall(bytes(data, "utf-8"))
+       sock.sendall(b"\n")
 
        # Receive data from the server and shut down
        received = str(sock.recv(1024), "utf-8")
 
-   print("Sent:     {}".format(data))
-   print("Received: {}".format(received))
+   print("Sent:    ", data)
+   print("Received:", received)
 
 
 The output of the example should look something like this:
@@ -599,7 +610,7 @@ This is the server side::
        def handle(self):
            data = self.request[0].strip()
            socket = self.request[1]
-           print("{} wrote:".format(self.client_address[0]))
+           print(f"{self.client_address[0]} wrote:")
            print(data)
            socket.sendto(data.upper(), self.client_address)
 
@@ -624,8 +635,8 @@ This is the client side::
    sock.sendto(bytes(data + "\n", "utf-8"), (HOST, PORT))
    received = str(sock.recv(1024), "utf-8")
 
-   print("Sent:     {}".format(data))
-   print("Received: {}".format(received))
+   print("Sent:    ", data)
+   print("Received:", received)
 
 The output of the example should look exactly like for the TCP server example.
 

Doc/tools/extensions/implementation_detail.py

@@ -0,0 +1,47 @@
+"""Support for marking up implementation details."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from docutils import nodes
+from sphinx.locale import _ as sphinx_gettext
+from sphinx.util.docutils import SphinxDirective
+
+if TYPE_CHECKING:
+    from sphinx.application import Sphinx
+    from sphinx.util.typing import ExtensionMetadata
+
+
+class ImplementationDetail(SphinxDirective):
+    has_content = True
+    final_argument_whitespace = True
+
+    # This text is copied to templates/dummy.html
+    label_text = sphinx_gettext("CPython implementation detail:")
+
+    def run(self):
+        self.assert_has_content()
+        content_nodes = self.parse_content_to_nodes()
+
+        # insert our prefix at the start of the first paragraph
+        first_node = content_nodes[0]
+        first_node[:0] = [
+            nodes.strong(self.label_text, self.label_text),
+            nodes.Text(" "),
+        ]
+
+        # create a new compound container node
+        cnode = nodes.compound("", *content_nodes, classes=["impl-detail"])
+        self.set_source_info(cnode)
+        return [cnode]
+
+
+def setup(app: Sphinx) -> ExtensionMetadata:
+    app.add_directive("impl-detail", ImplementationDetail)
+
+    return {
+        "version": "1.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

Doc/tools/extensions/pyspecific.py

@@ -65,31 +65,6 @@ def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
     return [refnode], []
 
 
-# Support for marking up implementation details
-
-class ImplementationDetail(SphinxDirective):
-
-    has_content = True
-    final_argument_whitespace = True
-
-    # This text is copied to templates/dummy.html
-    label_text = sphinx_gettext('CPython implementation detail:')
-
-    def run(self):
-        self.assert_has_content()
-        pnode = nodes.compound(classes=['impl-detail'])
-        content = self.content
-        add_text = nodes.strong(self.label_text, self.label_text)
-        self.state.nested_parse(content, self.content_offset, pnode)
-        content = nodes.inline(pnode[0].rawsource, translatable=True)
-        content.source = pnode[0].source
-        content.line = pnode[0].line
-        content += pnode[0].children
-        pnode[0].replace_self(nodes.paragraph(
-            '', '', add_text, nodes.Text(' '), content, translatable=False))
-        return [pnode]
-
-
 class PyCoroutineMixin(object):
     def handle_signature(self, sig, signode):
         ret = super(PyCoroutineMixin, self).handle_signature(sig, signode)
@@ -219,7 +194,6 @@ def patch_pairindextypes(app, _env) -> None:
 def setup(app):
     app.add_role('issue', issue_role)
     app.add_role('gh', gh_issue_role)
-    app.add_directive('impl-detail', ImplementationDetail)
     app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature)
     app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command)
     app.add_object_type('monitoring-event', 'monitoring-event', '%s (monitoring event)', parse_monitoring_event)

Doc/tools/templates/dummy.html

@@ -1,12 +1,6 @@
 This file is not an actual template, but used to add some
 texts in extensions to sphinx.pot file.
 
-In extensions/pyspecific.py:
-
-{% trans %}CPython implementation detail:{% endtrans %}
-{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %}
-{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %}
-
 In extensions/availability.py:
 
 {% trans %}Availability{% endtrans %}
@@ -27,6 +21,15 @@
 {% trans %}Return value: New reference.{% endtrans %}
 {% trans %}Return value: Borrowed reference.{% endtrans %}
 
+In extensions/implementation_detail.py:
+
+{% trans %}CPython implementation detail:{% endtrans %}
+
+In extensions/changes.py:
+
+{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %}
+{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %}
+
 In docsbuild-scripts, when rewriting indexsidebar.html with actual versions:
 
 {% trans %}in development{% endtrans %}

Doc/using/configure.rst

@@ -618,6 +618,16 @@ also be used to improve performance.
    Enable computed gotos in evaluation loop (enabled by default on supported
    compilers).
 
+.. option:: --with-tail-call-interp
+
+   Enable interpreters using tail calls in CPython. If enabled, enabling PGO
+   (:option:`--enable-optimizations`) is highly recommended. This option specifically
+   requires a C compiler with proper tail call support, and the
+   `preserve_none <https://clang.llvm.org/docs/AttributeReference.html#preserve-none>`_
+   calling convention. For example, Clang 19 and newer supports this feature.
+
+   .. versionadded:: next
+
 .. option:: --without-mimalloc
 
    Disable the fast :ref:`mimalloc <mimalloc>` allocator