Merge branch 'main' into pyrepl/completions-below

Author: danielhollas

.devcontainer/Dockerfile

@@ -6,7 +6,7 @@ ENV WASI_SDK_VERSION=21
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
-ENV WASMTIME_VERSION=18.0.3
+ENV WASMTIME_VERSION=22.0.0
 ENV WASMTIME_CPU_ARCH=x86_64
 
 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \

.github/CODEOWNERS

@@ -72,6 +72,7 @@ Include/internal/pycore_freelist.h  @ericsnowcurrently
 Include/internal/pycore_global_objects.h  @ericsnowcurrently
 Include/internal/pycore_obmalloc.h  @ericsnowcurrently
 Include/internal/pycore_pymem.h     @ericsnowcurrently
+Include/internal/pycore_stackref.h  @Fidget-Spinner
 Modules/main.c                @ericsnowcurrently
 Programs/_bootstrap_python.c  @ericsnowcurrently
 Programs/python.c             @ericsnowcurrently

.github/workflows/build.yml

@@ -179,18 +179,24 @@ jobs:
         run: make check-c-globals
 
   build_windows:
-    name: 'Windows'
+    name: >-
+      Windows
+      ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }}
     needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
-    uses: ./.github/workflows/reusable-windows.yml
-
-  build_windows_free_threading:
-    name: 'Windows (free-threading)'
-    needs: check_source
-    if: needs.check_source.outputs.run_tests == 'true'
+    if: fromJSON(needs.check_source.outputs.run_tests)
+    strategy:
+      matrix:
+        arch:
+        - Win32
+        - x64
+        - arm64
+        free-threading:
+        - false
+        - true
     uses: ./.github/workflows/reusable-windows.yml
     with:
-      free-threading: true
+      arch: ${{ matrix.arch }}
+      free-threading: ${{ matrix.free-threading }}
 
   build_macos:
     name: 'macOS'
@@ -393,7 +399,7 @@ jobs:
         path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/
         key: hypothesis-database-${{ github.head_ref || github.run_id }}
         restore-keys: |
-          - hypothesis-database-
+          hypothesis-database-
     - name: "Run tests"
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -556,7 +562,6 @@ jobs:
     - build_ubuntu_ssltests
     - build_wasi
     - build_windows
-    - build_windows_free_threading
     - test_hypothesis
     - build_asan
     - build_tsan
@@ -592,7 +597,6 @@ jobs:
             build_ubuntu_ssltests,
             build_wasi,
             build_windows,
-            build_windows_free_threading,
             build_asan,
             build_tsan,
             build_tsan_free_threading,

.github/workflows/jit.yml

@@ -133,17 +133,15 @@ jobs:
           make all --jobs 4
           ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Native Linux
         if: runner.os == 'Linux' && matrix.architecture == 'x86_64'
         run: |
           sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }}
           export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH"
-          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations' }}
+          ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }}
           make all --jobs 4
           ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
-      # --with-lto has been removed temporarily as a result of an open issue in LLVM 18 (see https://github.com/llvm/llvm-project/issues/87553)
       - name: Emulated Linux
         if: runner.os == 'Linux' && matrix.architecture != 'x86_64'
         # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux.
@@ -161,7 +159,7 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations ' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.github/workflows/reusable-wasi.yml

@@ -11,7 +11,7 @@ jobs:
     timeout-minutes: 60
     runs-on: ubuntu-22.04
     env:
-      WASMTIME_VERSION: 18.0.3
+      WASMTIME_VERSION: 22.0.0
       WASI_SDK_VERSION: 21
       WASI_SDK_PATH: /opt/wasi-sdk
       CROSS_BUILD_PYTHON: cross-build/build
@@ -20,9 +20,9 @@ jobs:
     - uses: actions/checkout@v4
     # No problem resolver registered as one doesn't currently exist for Clang.
     - name: "Install wasmtime"
-      uses: jcbhmr/setup-wasmtime@v2
+      uses: bytecodealliance/actions/wasmtime/setup@v1
       with:
-        wasmtime-version: ${{ env.WASMTIME_VERSION }}
+        version: ${{ env.WASMTIME_VERSION }}
     - name: "Restore WASI SDK"
       id: cache-wasi-sdk
       uses: actions/cache@v4
@@ -50,8 +50,10 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache
-        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
+        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python.
+        # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables.
+        # (Make sure to keep the key in sync with the other config.cache step below.)
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
     - name: "Configure build Python"
       run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug
     - name: "Make build Python"
@@ -60,8 +62,8 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ${{ env.CROSS_BUILD_WASI }}/config.cache
-        # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python
-        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-wasi-sdk-${{ env.WASI_SDK_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }}
+        # Should be kept in sync with the other config.cache step above.
+        key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }}
     - name: "Configure host"
       # `--with-pydebug` inferred from configure-build-python
       run: python3 Tools/wasm/wasi.py configure-host -- --config-cache

.github/workflows/reusable-windows.yml

@@ -1,53 +1,45 @@
 on:
   workflow_call:
     inputs:
+      arch:
+        description: CPU architecture
+        required: true
+        type: string
       free-threading:
+        description: Whether to use no-GIL mode
         required: false
         type: boolean
         default: false
 
-jobs:
-  build_win32:
-    name: 'build and test (x86)'
-    runs-on: windows-latest
-    timeout-minutes: 60
-    env:
-      IncludeUwp: 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -v -p Win32 ${{ inputs.free-threading && '--disable-gil' || '' }}
-    - name: Display build info
-      run: .\python.bat -m test.pythoninfo
-    - name: Tests
-      run: .\PCbuild\rt.bat -p Win32 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }}
+env:
+  IncludeUwp: >-
+    true
 
-  build_win_amd64:
-    name: 'build and test (x64)'
+jobs:
+  build:
+    name: >-
+      build${{ inputs.arch != 'arm64' && ' and test' || '' }}
+      (${{ inputs.arch }})
     runs-on: windows-latest
     timeout-minutes: 60
-    env:
-       IncludeUwp: 'true'
     steps:
     - uses: actions/checkout@v4
     - name: Register MSVC problem matcher
+      if: inputs.arch != 'Win32'
       run: echo "::add-matcher::.github/problem-matchers/msvc.json"
     - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -v -p x64 ${{ inputs.free-threading && '--disable-gil' || '' }}
+      run: >-
+        .\PCbuild\build.bat
+        -e -d -v
+        -p ${{ inputs.arch }}
+        ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Display build info
+      if: inputs.arch != 'arm64'
       run: .\python.bat -m test.pythoninfo
     - name: Tests
-      run: .\PCbuild\rt.bat -p x64 -d -q --fast-ci ${{ inputs.free-threading && '--disable-gil' || '' }}
-
-  build_win_arm64:
-    name: 'build (arm64)'
-    runs-on: windows-latest
-    timeout-minutes: 60
-    env:
-      IncludeUwp: 'true'
-    steps:
-    - uses: actions/checkout@v4
-    - name: Register MSVC problem matcher
-      run: echo "::add-matcher::.github/problem-matchers/msvc.json"
-    - name: Build CPython
-      run: .\PCbuild\build.bat -e -d -v -p arm64 ${{ inputs.free-threading && '--disable-gil' || '' }}
+      if: inputs.arch != 'arm64'
+      run: >-
+        .\PCbuild\rt.bat
+        -p ${{ inputs.arch }}
+        -d -q --fast-ci
+        ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}

Doc/README.rst

@@ -28,7 +28,7 @@ install the tools into there.
 Using make
 ----------
 
-To get started on UNIX, you can create a virtual environment and build
+To get started on Unix, you can create a virtual environment and build
 documentation with the commands::
 
   make venv
@@ -40,13 +40,13 @@ If you'd like to create the virtual environment in a different location,
 you can specify it using the ``VENVDIR`` variable.
 
 You can also skip creating the virtual environment altogether, in which case
-the Makefile will look for instances of ``sphinx-build`` and ``blurb``
+the ``Makefile`` will look for instances of ``sphinx-build`` and ``blurb``
 installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and
 ``BLURB`` variables).
 
-On Windows, we try to emulate the Makefile as closely as possible with a
+On Windows, we try to emulate the ``Makefile`` as closely as possible with a
 ``make.bat`` file. If you need to specify the Python interpreter to use,
-set the PYTHON environment variable.
+set the ``PYTHON`` environment variable.
 
 Available make targets are:
 
@@ -62,15 +62,19 @@ Available make targets are:
 * "htmlview", which re-uses the "html" builder, but then opens the main page
   in your default web browser.
 
+* "htmllive", which re-uses the "html" builder, rebuilds the docs,
+  starts a local server, and automatically reloads the page in your browser
+  when you make changes to reST files (Unix only).
+
 * "htmlhelp", which builds HTML files and a HTML Help project file usable to
   convert them into a single Compiled HTML (.chm) file -- these are popular
   under Microsoft Windows, but very handy on every platform.
 
   To create the CHM file, you need to run the Microsoft HTML Help Workshop
-  over the generated project (.hhp) file.  The make.bat script does this for
+  over the generated project (.hhp) file.  The ``make.bat`` script does this for
   you on Windows.
 
-* "latex", which builds LaTeX source files as input to "pdflatex" to produce
+* "latex", which builds LaTeX source files as input to ``pdflatex`` to produce
   PDF documents.
 
 * "text", which builds a plain text file for each source file.
@@ -95,8 +99,6 @@ Available make targets are:
 
 * "check", which checks for frequent markup errors.
 
-* "serve", which serves the build/html directory on port 8000.
-
 * "dist", (Unix only) which creates distributable archives of HTML, text,
   PDF, and EPUB builds.
 

Doc/c-api/cell.rst

@@ -39,7 +39,8 @@ Cell objects are not likely to be useful elsewhere.
 
 .. c:function:: PyObject* PyCell_Get(PyObject *cell)
 
-   Return the contents of the cell *cell*.
+   Return the contents of the cell *cell*, which can be ``NULL``.
+   If *cell* is not a cell object, returns ``NULL`` with an exception set.
 
 
 .. c:function:: PyObject* PyCell_GET(PyObject *cell)
@@ -52,8 +53,10 @@ Cell objects are not likely to be useful elsewhere.
 
    Set the contents of the cell object *cell* to *value*.  This releases the
    reference to any current content of the cell. *value* may be ``NULL``.  *cell*
-   must be non-``NULL``; if it is not a cell object, ``-1`` will be returned.  On
-   success, ``0`` will be returned.
+   must be non-``NULL``.
+
+   On success, return ``0``.
+   If *cell* is not a cell object, set an exception and return ``-1``.
 
 
 .. c:function:: void PyCell_SET(PyObject *cell, PyObject *value)

Doc/c-api/dict.rst

@@ -156,7 +156,7 @@ Dictionary Objects
 
 .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result)
 
-   Similar than :c:func:`PyDict_GetItemRef`, but *key* is specified as a
+   Similar to :c:func:`PyDict_GetItemRef`, but *key* is specified as a
    :c:expr:`const char*` UTF-8 encoded bytes string, rather than a
    :c:expr:`PyObject*`.
 
@@ -206,7 +206,7 @@ Dictionary Objects
      ``NULL``, and return ``0``.
    - On error, raise an exception and return ``-1``.
 
-   This is similar to :meth:`dict.pop`, but without the default value and
+   Similar to :meth:`dict.pop`, but without the default value and
    not raising :exc:`KeyError` if the key missing.
 
    .. versionadded:: 3.13
@@ -290,6 +290,17 @@ Dictionary Objects
           Py_DECREF(o);
       }
 
+   The function is not thread-safe in the :term:`free-threaded <free threading>`
+   build without external synchronization.  You can use
+   :c:macro:`Py_BEGIN_CRITICAL_SECTION` to lock the dictionary while iterating
+   over it::
+
+      Py_BEGIN_CRITICAL_SECTION(self->dict);
+      while (PyDict_Next(self->dict, &pos, &key, &value)) {
+          ...
+      }
+      Py_END_CRITICAL_SECTION();
+
 
 .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override)
 

Doc/c-api/long.rst

@@ -405,14 +405,13 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
 
    Passing zero to *n_bytes* will return the size of a buffer that would
    be large enough to hold the value. This may be larger than technically
-   necessary, but not unreasonably so.
+   necessary, but not unreasonably so. If *n_bytes=0*, *buffer* may be
+   ``NULL``.
 
    .. note::
 
       Passing *n_bytes=0* to this function is not an accurate way to determine
-      the bit length of a value.
-
-   If *n_bytes=0*, *buffer* may be ``NULL``.
+      the bit length of the value.
 
    To get at the entire Python value of an unknown size, the function can be
    called twice: first to determine the buffer size, then to fill it::
@@ -462,6 +461,7 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    .. c:macro:: Py_ASNATIVEBYTES_NATIVE_ENDIAN   ``3``
    .. c:macro:: Py_ASNATIVEBYTES_UNSIGNED_BUFFER ``4``
    .. c:macro:: Py_ASNATIVEBYTES_REJECT_NEGATIVE ``8``
+   .. c:macro:: Py_ASNATIVEBYTES_ALLOW_INDEX     ``16``
    ============================================= ======
 
    Specifying ``Py_ASNATIVEBYTES_NATIVE_ENDIAN`` will override any other endian
@@ -483,6 +483,13 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    provided there is enough space for at least one sign bit, regardless of
    whether ``Py_ASNATIVEBYTES_UNSIGNED_BUFFER`` was specified.
 
+   If ``Py_ASNATIVEBYTES_ALLOW_INDEX`` is specified and a non-integer value is
+   passed, its :meth:`~object.__index__` method will be called first. This may
+   result in Python code executing and other threads being allowed to run, which
+   could cause changes to other objects or values in use. When *flags* is
+   ``-1``, this option is not set, and non-integer values will raise
+   :exc:`TypeError`.
+
    .. note::
 
       With the default *flags* (``-1``, or *UNSIGNED_BUFFER*  without