Merge branch 'main' into SSL_sendfile

Author: illia-v

.github/CODEOWNERS

@@ -188,7 +188,7 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 
 # AST
 Python/ast.c                  @isidentical @JelleZijlstra @eclips4
-Python/ast_opt.c              @isidentical @eclips4
+Python/ast_preprocess.c       @isidentical @eclips4
 Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
 Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
 Lib/ast.py                    @isidentical @JelleZijlstra @eclips4

.github/workflows/jit.yml

@@ -95,10 +95,10 @@ jobs:
         with:
           python-version: '3.11'
 
+      # PCbuild downloads LLVM automatically:
       - name: Windows
         if: runner.os == 'Windows'
         run: |
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
@@ -126,29 +126,30 @@ jobs:
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-  jit-with-disabled-gil:
-    name: Free-Threaded (Debug)
-    needs: interpreter
-    runs-on: ubuntu-24.04
-    timeout-minutes: 90
-    strategy:
-      fail-fast: false
-      matrix:
-        llvm:
-          - 19
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
-      - name: Build with JIT enabled and GIL disabled
-        run: |
-          sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
-          export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit --with-pydebug --disable-gil
-          make all --jobs 4
-      - name: Run tests
-        run: |
-          ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
+  # XXX: GH-133171
+  # jit-with-disabled-gil:
+  #   name: Free-Threaded (Debug)
+  #   needs: interpreter
+  #   runs-on: ubuntu-24.04
+  #   timeout-minutes: 90
+  #   strategy:
+  #     fail-fast: false
+  #     matrix:
+  #       llvm:
+  #         - 19
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #       with:
+  #         persist-credentials: false
+  #     - uses: actions/setup-python@v5
+  #       with:
+  #         python-version: '3.11'
+  #     - name: Build with JIT enabled and GIL disabled
+  #       run: |
+  #         sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
+  #         export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
+  #         ./configure --enable-experimental-jit --with-pydebug --disable-gil
+  #         make all --jobs 4
+  #     - name: Run tests
+  #       run: |
+  #         ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3

.github/workflows/reusable-context.yml

@@ -97,6 +97,8 @@ jobs:
       run: python Tools/build/compute-changes.py
       env:
         GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+        CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }}
+        CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }}
 
     - name: Compute hash for config cache key
       id: config-hash

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.11.6
+    rev: v0.11.8
     hooks:
       - id: ruff
         name: Run Ruff (lint) on Doc/
@@ -22,14 +22,14 @@ repos:
         name: Run Ruff (format) on Doc/
         args: [--check]
         files: ^Doc/
+      - id: ruff-format
+        name: Run Ruff (format) on Tools/build/check_warnings.py
+        args: [--check, --config=Tools/build/.ruff.toml]
+        files: ^Tools/build/check_warnings.py
 
   - repo: https://github.com/psf/black-pre-commit-mirror
     rev: 25.1.0
     hooks:
-      - id: black
-        name: Run Black on Tools/build/check_warnings.py
-        files: ^Tools/build/check_warnings.py
-        args: [--line-length=79]
       - id: black
         name: Run Black on Tools/jit/
         files: ^Tools/jit/

Doc/deprecations/index.rst

@@ -7,6 +7,8 @@ Deprecations
 
 .. include:: pending-removal-in-3.17.rst
 
+.. include:: pending-removal-in-3.19.rst
+
 .. include:: pending-removal-in-future.rst
 
 C API deprecations

Doc/deprecations/pending-removal-in-3.19.rst

@@ -0,0 +1,8 @@
+Pending removal in Python 3.19
+------------------------------
+
+* :mod:`ctypes`:
+
+  * Implicitly switching to the MSVC-compatible struct layout by setting
+    :attr:`~ctypes.Structure._pack_` but not :attr:`~ctypes.Structure._layout_`
+    on non-Windows platforms.

Doc/howto/regex.rst

@@ -738,9 +738,12 @@ given location, they can obviously be matched an infinite number of times.
    different: ``\A`` still matches only at the beginning of the string, but ``^``
    may match at any location inside the string that follows a newline character.
 
-``\Z``
+``\z``
    Matches only at the end of the string.
 
+``\Z``
+   The same as ``\z``.  For compatibility with old Python versions.
+
 ``\b``
    Word boundary.  This is a zero-width assertion that matches only at the
    beginning or end of a word.  A word is defined as a sequence of alphanumeric

Doc/library/annotationlib.rst

@@ -40,7 +40,7 @@ The :func:`get_annotations` function is the main entry point for
 retrieving annotations. Given a function, class, or module, it returns
 an annotations dictionary in the requested format. This module also provides
 functionality for working directly with the :term:`annotate function`
-that is used to evaluate annotations, such as :func:`get_annotate_function`
+that is used to evaluate annotations, such as :func:`get_annotate_from_class_namespace`
 and :func:`call_annotate_function`, as well as the
 :func:`call_evaluate_function` function for working with
 :term:`evaluate functions <evaluate function>`.
@@ -132,7 +132,7 @@ Classes
 
       Values are real annotation values (as per :attr:`Format.VALUE` format)
       for defined values, and :class:`ForwardRef` proxies for undefined
-      values. Real objects may contain references to, :class:`ForwardRef`
+      values. Real objects may contain references to :class:`ForwardRef`
       proxy objects.
 
    .. attribute:: STRING
@@ -172,14 +172,21 @@ Classes
       :class:`~ForwardRef`. The string may not be exactly equivalent
       to the original source.
 
-   .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None)
+   .. method:: evaluate(*, owner=None, globals=None, locals=None, type_params=None, format=Format.VALUE)
 
       Evaluate the forward reference, returning its value.
 
-      This may throw an exception, such as :exc:`NameError`, if the forward
+      If the *format* argument is :attr:`~Format.VALUE` (the default),
+      this method may throw an exception, such as :exc:`NameError`, if the forward
       reference refers to a name that cannot be resolved. The arguments to this
       method can be used to provide bindings for names that would otherwise
-      be undefined.
+      be undefined. If the *format* argument is :attr:`~Format.FORWARDREF`,
+      the method will never throw an exception, but may return a :class:`~ForwardRef`
+      instance. For example, if the forward reference object contains the code
+      ``list[undefined]``, where ``undefined`` is a name that is not defined,
+      evaluating it with the :attr:`~Format.FORWARDREF` format will return
+      ``list[ForwardRef('undefined')]``. If the *format* argument is
+      :attr:`~Format.STRING`, the method will return :attr:`~ForwardRef.__forward_arg__`.
 
       The *owner* parameter provides the preferred mechanism for passing scope
       information to this method. The owner of a :class:`~ForwardRef` is the
@@ -300,15 +307,13 @@ Functions
 
    .. versionadded:: 3.14
 
-.. function:: get_annotate_function(obj)
+.. function:: get_annotate_from_class_namespace(namespace)
 
-   Retrieve the :term:`annotate function` for *obj*. Return :const:`!None`
-   if *obj* does not have an annotate function. *obj* may be a class, function,
-   module, or a namespace dictionary for a class. The last case is useful during
-   class creation, e.g. in the ``__new__`` method of a metaclass.
-
-   This is usually equivalent to accessing the :attr:`~object.__annotate__`
-   attribute of *obj*, but access through this public function is preferred.
+   Retrieve the :term:`annotate function` from a class namespace dictionary *namespace*.
+   Return :const:`!None` if the namespace does not contain an annotate function.
+   This is primarily useful before the class has been fully created (e.g., in a metaclass);
+   after the class exists, the annotate function can be retrieved with ``cls.__annotate__``.
+   See :ref:`below <annotationlib-metaclass>` for an example using this function in a metaclass.
 
    .. versionadded:: 3.14
 
@@ -407,3 +412,76 @@ Functions
 
    .. versionadded:: 3.14
 
+
+Recipes
+-------
+
+.. _annotationlib-metaclass:
+
+Using annotations in a metaclass
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+A :ref:`metaclass <metaclasses>` may want to inspect or even modify the annotations
+in a class body during class creation. Doing so requires retrieving annotations
+from the class namespace dictionary. For classes created with
+``from __future__ import annotations``, the annotations will be in the ``__annotations__``
+key of the dictionary. For other classes with annotations,
+:func:`get_annotate_from_class_namespace` can be used to get the
+annotate function, and :func:`call_annotate_function` can be used to call it and
+retrieve the annotations. Using the :attr:`~Format.FORWARDREF` format will usually
+be best, because this allows the annotations to refer to names that cannot yet be
+resolved when the class is created.
+
+To modify the annotations, it is best to create a wrapper annotate function
+that calls the original annotate function, makes any necessary adjustments, and
+returns the result.
+
+Below is an example of a metaclass that filters out all :class:`typing.ClassVar`
+annotations from the class and puts them in a separate attribute:
+
+.. code-block:: python
+
+   import annotationlib
+   import typing
+
+   class ClassVarSeparator(type):
+      def __new__(mcls, name, bases, ns):
+         if "__annotations__" in ns:  # from __future__ import annotations
+            annotations = ns["__annotations__"]
+            classvar_keys = {
+               key for key, value in annotations.items()
+               # Use string comparison for simplicity; a more robust solution
+               # could use annotationlib.ForwardRef.evaluate
+               if value.startswith("ClassVar")
+            }
+            classvars = {key: annotations[key] for key in classvar_keys}
+            ns["__annotations__"] = {
+               key: value for key, value in annotations.items()
+               if key not in classvar_keys
+            }
+            wrapped_annotate = None
+         elif annotate := annotationlib.get_annotate_from_class_namespace(ns):
+            annotations = annotationlib.call_annotate_function(
+               annotate, format=annotationlib.Format.FORWARDREF
+            )
+            classvar_keys = {
+               key for key, value in annotations.items()
+               if typing.get_origin(value) is typing.ClassVar
+            }
+            classvars = {key: annotations[key] for key in classvar_keys}
+
+            def wrapped_annotate(format):
+               annos = annotationlib.call_annotate_function(annotate, format, owner=typ)
+               return {key: value for key, value in annos.items() if key not in classvar_keys}
+
+         else:  # no annotations
+            classvars = {}
+            wrapped_annotate = None
+         typ = super().__new__(mcls, name, bases, ns)
+
+         if wrapped_annotate is not None:
+            # Wrap the original __annotate__ with a wrapper that removes ClassVars
+            typ.__annotate__ = wrapped_annotate
+         typ.classvars = classvars  # Store the ClassVars in a separate attribute
+         return typ
+

Doc/library/argparse.rst

@@ -74,7 +74,7 @@ ArgumentParser objects
                           prefix_chars='-', fromfile_prefix_chars=None, \
                           argument_default=None, conflict_handler='error', \
                           add_help=True, allow_abbrev=True, exit_on_error=True, \
-                          suggest_on_error=False, color=False)
+                          *, suggest_on_error=False, color=False)
 
    Create a new :class:`ArgumentParser` object. All parameters should be passed
    as keyword arguments. Each parameter has its own more detailed description

Doc/library/asyncio-eventloop.rst

@@ -361,7 +361,7 @@ Creating Futures and Tasks
 
    .. versionadded:: 3.5.2
 
-.. method:: loop.create_task(coro, *, name=None, context=None)
+.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None)
 
    Schedule the execution of :ref:`coroutine <coroutine>` *coro*.
    Return a :class:`Task` object.
@@ -377,12 +377,20 @@ Creating Futures and Tasks
    custom :class:`contextvars.Context` for the *coro* to run in.
    The current context copy is created when no *context* is provided.
 
+   An optional keyword-only *eager_start* argument allows specifying
+   if the task should execute eagerly during the call to create_task,
+   or be scheduled later. If *eager_start* is not passed the mode set
+   by :meth:`loop.set_task_factory` will be used.
+
    .. versionchanged:: 3.8
       Added the *name* parameter.
 
    .. versionchanged:: 3.11
       Added the *context* parameter.
 
+   .. versionchanged:: next
+      Added the *eager_start* parameter.
+
 .. method:: loop.set_task_factory(factory)
 
    Set a task factory that will be used by