Catch up with main

Author: brandtbucher

.github/workflows/reusable-tsan.yml

@@ -74,6 +74,9 @@ jobs:
       run: make pythoninfo
     - name: Tests
       run: ./python -m test --tsan -j4
+    - name: Parallel tests
+      if: fromJSON(inputs.free-threading)
+      run: ./python -m test --tsan-parallel --parallel-threads=4 -j4
     - name: Display TSAN logs
       if: always()
       run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000

.github/workflows/tail-call.yml

@@ -0,0 +1,113 @@
+name: Tail calling interpreter
+on:
+  pull_request:
+    paths:
+      - 'Python/bytecodes.c'
+      - 'Python/ceval.c'
+      - 'Python/ceval_macros.h'
+  push:
+    paths:
+      - 'Python/bytecodes.c'
+      - 'Python/ceval.c'
+      - 'Python/ceval_macros.h'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+env:
+  FORCE_COLOR: 1
+
+jobs:
+  tail-call:
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 90
+    strategy:
+      fail-fast: false
+      matrix:
+        target:
+# Un-comment as we add support for more platforms for tail-calling interpreters.
+#          - i686-pc-windows-msvc/msvc
+#          - x86_64-pc-windows-msvc/msvc
+#          - aarch64-pc-windows-msvc/msvc
+          - x86_64-apple-darwin/clang
+          - aarch64-apple-darwin/clang
+          - x86_64-unknown-linux-gnu/gcc
+          - aarch64-unknown-linux-gnu/gcc
+        llvm:
+          - 19
+        include:
+#          - target: i686-pc-windows-msvc/msvc
+#            architecture: Win32
+#            runner: windows-latest
+#          - target: x86_64-pc-windows-msvc/msvc
+#            architecture: x64
+#            runner: windows-latest
+#          - target: aarch64-pc-windows-msvc/msvc
+#            architecture: ARM64
+#            runner: windows-latest
+          - target: x86_64-apple-darwin/clang
+            architecture: x86_64
+            runner: macos-13
+          - target: aarch64-apple-darwin/clang
+            architecture: aarch64
+            runner: macos-14
+          - target: x86_64-unknown-linux-gnu/gcc
+            architecture: x86_64
+            runner: ubuntu-24.04
+          - target: aarch64-unknown-linux-gnu/gcc
+            architecture: aarch64
+            runner: ubuntu-24.04-arm
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Native Windows (debug)
+        if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }}
+          ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+      # No tests (yet):
+      - name: Emulated Windows (release)
+        if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
+        run: |
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
+          ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }}
+
+        # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966.
+        # This is a bug in the macOS runner image where the pre-installed Python is installed in the same
+        # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes
+        # the symlink to the pre-installed Python so that the Homebrew Python is used instead.
+      - name: Native macOS (debug)
+        if: runner.os == 'macOS'
+        run: |
+          brew update
+          find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete
+          brew install llvm@${{ matrix.llvm }}
+          export SDKROOT="$(xcrun --show-sdk-path)"
+          export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+          export PATH="/usr/local/opt/llvm/bin:$PATH"
+          CC=clang-19 ./configure --with-tail-call-interp --with-pydebug
+          make all --jobs 4
+          ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+
+      - name: Native Linux (release)
+        if: runner.os == 'Linux'
+        run: |
+          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+          CC=clang-19 ./configure --with-tail-call-interp
+          make all --jobs 4
+          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+

Doc/c-api/init.rst

@@ -1501,7 +1501,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
 
 .. c:function:: PyObject* PyUnstable_InterpreterState_GetMainModule(PyInterpreterState *interp)
 
-   Return a :term:`strong reference` to the ``__main__`` `module object <moduleobjects>`_
+   Return a :term:`strong reference` to the ``__main__`` :ref:`module object <moduleobjects>`
    for the given interpreter.
 
    The caller must hold the GIL.

Doc/conf.py

@@ -28,6 +28,7 @@
     'changes',
     'glossary_search',
     'grammar_snippet',
+    'implementation_detail',
     'lexers',
     'misc_news',
     'pydoc_topics',

Doc/library/imaplib.rst

@@ -10,6 +10,7 @@
 .. changes for IMAP4_SSL by Tino Lange <[email protected]>, March 2002
 .. changes for IMAP4_stream by Piers Lauder <[email protected]>,
    November 2002
+.. changes for IMAP4 IDLE by Forest <[email protected]>, August 2024
 
 **Source code:** :source:`Lib/imaplib.py`
 
@@ -187,7 +188,7 @@ However, the *password* argument to the ``LOGIN`` command is always quoted. If
 you want to avoid having an argument string quoted (eg: the *flags* argument to
 ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``).
 
-Each command returns a tuple: ``(type, [data, ...])`` where *type* is usually
+Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually
 ``'OK'`` or ``'NO'``, and *data* is either the text from the command response,
 or mandated results from the command. Each *data* is either a ``bytes``, or a
 tuple. If a tuple, then the first part is the header of the response, and the
@@ -307,6 +308,93 @@ An :class:`IMAP4` instance has the following methods:
    of the IMAP4 QUOTA extension defined in rfc2087.
 
 
+.. method:: IMAP4.idle(duration=None)
+
+   Return an :class:`!Idler`: an iterable context manager implementing the
+   IMAP4 ``IDLE`` command as defined in :rfc:`2177`.
+
+   The returned object sends the ``IDLE`` command when activated by the
+   :keyword:`with` statement, produces IMAP untagged responses via the
+   :term:`iterator` protocol, and sends ``DONE`` upon context exit.
+
+   All untagged responses that arrive after sending the ``IDLE`` command
+   (including any that arrive before the server acknowledges the command) will
+   be available via iteration. Any leftover responses (those not iterated in
+   the :keyword:`with` context) can be retrieved in the usual way after
+   ``IDLE`` ends, using :meth:`IMAP4.response`.
+
+   Responses are represented as ``(type, [data, ...])`` tuples, as described
+   in :ref:`IMAP4 Objects <imap4-objects>`.
+
+   The *duration* argument sets a maximum duration (in seconds) to keep idling,
+   after which any ongoing iteration will stop. It can be an :class:`int` or
+   :class:`float`, or ``None`` for no time limit.
+   Callers wishing to avoid inactivity timeouts on servers that impose them
+   should keep this at most 29 minutes (1740 seconds).
+   Requires a socket connection; *duration* must be ``None`` on
+   :class:`IMAP4_stream` connections.
+
+   .. code-block:: pycon
+
+      >>> with M.idle(duration=29 * 60) as idler:
+      ...     for typ, data in idler:
+      ...         print(typ, data)
+      ...
+      EXISTS [b'1']
+      RECENT [b'1']
+
+
+   .. method:: Idler.burst(interval=0.1)
+
+      Yield a burst of responses no more than *interval* seconds apart
+      (expressed as an :class:`int` or :class:`float`).
+
+      This :term:`generator` is an alternative to iterating one response at a
+      time, intended to aid in efficient batch processing. It retrieves the
+      next response along with any immediately available subsequent responses.
+      (For example, a rapid series of ``EXPUNGE`` responses after a bulk
+      delete.)
+
+      Requires a socket connection; does not work on :class:`IMAP4_stream`
+      connections.
+
+      .. code-block:: pycon
+
+         >>> with M.idle() as idler:
+         ...     # get a response and any others following by < 0.1 seconds
+         ...     batch = list(idler.burst())
+         ...     print(f'processing {len(batch)} responses...')
+         ...     print(batch)
+         ...
+         processing 3 responses...
+         [('EXPUNGE', [b'2']), ('EXPUNGE', [b'1']), ('RECENT', [b'0'])]
+
+      .. tip::
+
+         The ``IDLE`` context's maximum duration, as passed to
+         :meth:`IMAP4.idle`, is respected when waiting for the first response
+         in a burst. Therefore, an expired :class:`!Idler` will cause this
+         generator to return immediately without producing anything. Callers
+         should consider this if using it in a loop.
+
+
+   .. note::
+
+      The iterator returned by :meth:`IMAP4.idle` is usable only within a
+      :keyword:`with` statement. Before or after that context, unsolicited
+      responses are collected internally whenever a command finishes, and can
+      be retrieved with :meth:`IMAP4.response`.
+
+   .. note::
+
+      The :class:`!Idler` class name and structure are internal interfaces,
+      subject to change. Calling code can rely on its context management,
+      iteration, and public method to remain stable, but should not subclass,
+      instantiate, compare, or otherwise directly reference the class.
+
+   .. versionadded:: next
+
+
 .. method:: IMAP4.list([directory[, pattern]])
 
    List mailbox names in *directory* matching *pattern*.  *directory* defaults to

Doc/library/itertools.rst

@@ -838,10 +838,10 @@ and :term:`generators <generator>` which incur interpreter overhead.
 
 .. testcode::
 
-   from collections import deque
+   from collections import Counter, deque
    from contextlib import suppress
    from functools import reduce
-   from math import sumprod, isqrt
+   from math import comb, prod, sumprod, isqrt
    from operator import itemgetter, getitem, mul, neg
 
    def take(n, iterable):
@@ -1127,6 +1127,12 @@ The following recipes have a more mathematical flavor:
            n -= n // prime
        return n
 
+   def multinomial(*counts):
+       "Number of distinct arrangements of a multiset."
+       # Counter('abracadabra').values() -> 5 2 1 1 2
+       # multinomial(5, 2, 1, 1, 2) → 83160
+       return prod(map(comb, accumulate(counts), counts))
+
 
 .. doctest::
     :hide:
@@ -1730,6 +1736,12 @@ The following recipes have a more mathematical flavor:
     >>> ''.join(it)
     'DEF1'
 
+    >>> multinomial(5, 2, 1, 1, 2)
+    83160
+    >>> word = 'coffee'
+    >>> multinomial(*Counter(word).values()) == len(set(permutations(word)))
+    True
+
 
 .. testcode::
     :hide:

Doc/library/socket.rst

@@ -1668,11 +1668,6 @@ to sockets.
    See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument
    *flags*; it defaults to zero.
 
-   .. note::
-
-      For best match with hardware and network realities, the value of  *bufsize*
-      should be a relatively small power of 2, for example, 4096.
-
    .. versionchanged:: 3.5
       If the system call is interrupted and the signal handler does not raise
       an exception, the method now retries the system call instead of raising

Doc/library/socketserver.rst

@@ -499,11 +499,17 @@ This is the server side::
 
        def handle(self):
            # self.request is the TCP socket connected to the client
-           self.data = self.request.recv(1024).strip()
-           print("Received from {}:".format(self.client_address[0]))
-           print(self.data)
+           pieces = [b'']
+           total = 0
+           while b'\n' not in pieces[-1] and total < 10_000:
+               pieces.append(self.request.recv(2000))
+               total += len(pieces[-1])
+           self.data = b''.join(pieces)
+           print(f"Received from {self.client_address[0]}:")
+           print(self.data.decode("utf-8"))
            # just send back the same data, but upper-cased
            self.request.sendall(self.data.upper())
+           # after we return, the socket will be closed.
 
    if __name__ == "__main__":
        HOST, PORT = "localhost", 9999
@@ -520,20 +526,24 @@ objects that simplify communication by providing the standard file interface)::
    class MyTCPHandler(socketserver.StreamRequestHandler):
 
        def handle(self):
-           # self.rfile is a file-like object created by the handler;
-           # we can now use e.g. readline() instead of raw recv() calls
-           self.data = self.rfile.readline().strip()
-           print("{} wrote:".format(self.client_address[0]))
-           print(self.data)
+           # self.rfile is a file-like object created by the handler.
+           # We can now use e.g. readline() instead of raw recv() calls.
+           # We limit ourselves to 10000 bytes to avoid abuse by the sender.
+           self.data = self.rfile.readline(10000).rstrip()
+           print(f"{self.client_address[0]} wrote:")
+           print(self.data.decode("utf-8"))
            # Likewise, self.wfile is a file-like object used to write back
            # to the client
            self.wfile.write(self.data.upper())
 
 The difference is that the ``readline()`` call in the second handler will call
 ``recv()`` multiple times until it encounters a newline character, while the
-single ``recv()`` call in the first handler will just return what has been
-received so far from the client's ``sendall()`` call (typically all of it, but
-this is not guaranteed by the TCP protocol).
+the first handler had to use a ``recv()`` loop to accumulate data until a
+newline itself.  If it had just used a single ``recv()`` without the loop it
+would just have returned what has been received so far from the client.
+TCP is stream based: data arrives in the order it was sent, but there no
+correlation between client ``send()`` or ``sendall()`` calls and the number
+of ``recv()`` calls on the server required to receive it.
 
 
 This is the client side::
@@ -548,13 +558,14 @@ This is the client side::
    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
        # Connect to server and send data
        sock.connect((HOST, PORT))
-       sock.sendall(bytes(data + "\n", "utf-8"))
+       sock.sendall(bytes(data, "utf-8"))
+       sock.sendall(b"\n")
 
        # Receive data from the server and shut down
        received = str(sock.recv(1024), "utf-8")
 
-   print("Sent:     {}".format(data))
-   print("Received: {}".format(received))
+   print("Sent:    ", data)
+   print("Received:", received)
 
 
 The output of the example should look something like this:
@@ -599,7 +610,7 @@ This is the server side::
        def handle(self):
            data = self.request[0].strip()
            socket = self.request[1]
-           print("{} wrote:".format(self.client_address[0]))
+           print(f"{self.client_address[0]} wrote:")
            print(data)
            socket.sendto(data.upper(), self.client_address)
 
@@ -624,8 +635,8 @@ This is the client side::
    sock.sendto(bytes(data + "\n", "utf-8"), (HOST, PORT))
    received = str(sock.recv(1024), "utf-8")
 
-   print("Sent:     {}".format(data))
-   print("Received: {}".format(received))
+   print("Sent:    ", data)
+   print("Received:", received)
 
 The output of the example should look exactly like for the TCP server example.