Merge branch 'main' into feat/hashopenssl/refactoring-134531

Author: picnixz

.editorconfig

@@ -1,11 +1,11 @@
 root = true
 
-[*.{py,c,cpp,h,js,rst,md,yml,yaml}]
+[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}]
 trim_trailing_whitespace = true
 insert_final_newline = true
 indent_style = space
 
-[*.{py,c,cpp,h}]
+[*.{py,c,cpp,h,gram}]
 indent_size = 4
 
 [*.rst]

.github/CODEOWNERS

@@ -26,7 +26,8 @@ Modules/Setup*                @erlend-aasland
 **/*context*                  @1st1
 **/*genobject*                @markshannon
 **/*hamt*                     @1st1
-**/*jit*                      @brandtbucher @savannahostrowski
+**/*jit*                      @brandtbucher @savannahostrowski @diegorusso
+Python/perf_jit_trampoline.c  # Exclude the owners of "**/*jit*", above.
 Objects/set*                  @rhettinger
 Objects/dict*                 @methane @markshannon
 Objects/typevarobject.c       @JelleZijlstra

.github/workflows/build.yml

@@ -260,7 +260,7 @@ jobs:
       free-threading: ${{ matrix.free-threading }}
       os: ${{ matrix.os }}
 
-  build-ubuntu-ssltests:
+  build-ubuntu-ssltests-openssl:
     name: 'Ubuntu SSL tests with OpenSSL'
     runs-on: ${{ matrix.os }}
     timeout-minutes: 60
@@ -322,6 +322,81 @@ jobs:
     - name: SSL tests
       run: ./python Lib/test/ssltests.py
 
+  build-ubuntu-ssltests-awslc:
+    name: 'Ubuntu SSL tests with AWS-LC'
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    needs: build-context
+    if: needs.build-context.outputs.run-tests == 'true'
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04]
+        awslc_ver: [1.55.0]
+    env:
+      AWSLC_VER: ${{ matrix.awslc_ver}}
+      MULTISSL_DIR: ${{ github.workspace }}/multissl
+      OPENSSL_DIR: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}
+      LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/aws-lc/${{ matrix.awslc_ver }}/lib
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        persist-credentials: false
+    - name: Runner image version
+      run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV"
+    - name: Restore config.cache
+      uses: actions/cache@v4
+      with:
+        path: config.cache
+        key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}
+    - name: Register gcc problem matcher
+      run: echo "::add-matcher::.github/problem-matchers/gcc.json"
+    - name: Install dependencies
+      run: sudo ./.github/workflows/posix-deps-apt.sh
+    - name: Configure SSL lib env vars
+      run: |
+        echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV"
+        echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}" >> "$GITHUB_ENV"
+        echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/aws-lc/${AWSLC_VER}/lib" >> "$GITHUB_ENV"
+    - name: 'Restore AWS-LC build'
+      id: cache-aws-lc
+      uses: actions/cache@v4
+      with:
+        path: ./multissl/aws-lc/${{ matrix.awslc_ver }}
+        key: ${{ matrix.os }}-multissl-aws-lc-${{ matrix.awslc_ver }}
+    - name: Install AWS-LC
+      if: steps.cache-aws-lc.outputs.cache-hit != 'true'
+      run: |
+        python3 Tools/ssl/multissltests.py \
+          --steps=library \
+          --base-directory "$MULTISSL_DIR" \
+          --awslc ${{ matrix.awslc_ver }} \
+          --system Linux
+    - name: Add ccache to PATH
+      run: |
+        echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
+    - name: Configure ccache action
+      uses: hendrikmuhs/[email protected]
+      with:
+        save: false
+    - name: Configure CPython
+      run: |
+        ./configure CFLAGS="-fdiagnostics-format=json" \
+          --config-cache \
+          --enable-slower-safety \
+          --with-pydebug \
+          --with-openssl="$OPENSSL_DIR" \
+          --with-builtin-hashlib-hashes=blake2 \
+          --with-ssl-default-suites=openssl
+    - name: Build CPython
+      run: make -j
+    - name: Display build info
+      run: make pythoninfo
+    - name: Verify python is linked to AWS-LC
+      run: ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' | grep AWS-LC
+    - name: SSL tests
+      run: ./python Lib/test/ssltests.py
+
   build-wasi:
     name: 'WASI'
     needs: build-context
@@ -628,7 +703,8 @@ jobs:
     - build-windows-msi
     - build-macos
     - build-ubuntu
-    - build-ubuntu-ssltests
+    - build-ubuntu-ssltests-awslc
+    - build-ubuntu-ssltests-openssl
     - build-wasi
     - test-hypothesis
     - build-asan
@@ -643,7 +719,8 @@ jobs:
       with:
         allowed-failures: >-
           build-windows-msi,
-          build-ubuntu-ssltests,
+          build-ubuntu-ssltests-awslc,
+          build-ubuntu-ssltests-openssl,
           test-hypothesis,
           cifuzz,
         allowed-skips: >-
@@ -661,7 +738,8 @@ jobs:
             check-generated-files,
             build-macos,
             build-ubuntu,
-            build-ubuntu-ssltests,
+            build-ubuntu-ssltests-awslc,
+            build-ubuntu-ssltests-openssl,
             build-wasi,
             test-hypothesis,
             build-asan,

.github/workflows/jit.yml

@@ -5,6 +5,8 @@ on:
       - '**jit**'
       - 'Python/bytecodes.c'
       - 'Python/optimizer*.c'
+      - 'Python/executor_cases.c.h'
+      - 'Python/optimizer_cases.c.h'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'
@@ -13,6 +15,8 @@ on:
       - '**jit**'
       - 'Python/bytecodes.c'
       - 'Python/optimizer*.c'
+      - 'Python/executor_cases.c.h'
+      - 'Python/optimizer_cases.c.h'
       - '!Python/perf_jit_trampoline.c'
       - '!**/*.md'
       - '!**/*.ini'

.github/workflows/posix-deps-apt.sh

@@ -5,6 +5,7 @@ apt-get -yq install \
     build-essential \
     pkg-config \
     ccache \
+    cmake \
     gdb \
     lcov \
     libb2-dev \
@@ -25,3 +26,10 @@ apt-get -yq install \
     uuid-dev \
     xvfb \
     zlib1g-dev
+
+# Workaround missing libmpdec-dev on ubuntu 24.04:
+# https://launchpad.net/~ondrej/+archive/ubuntu/php
+# https://deb.sury.org/
+sudo add-apt-repository ppa:ondrej/php
+apt-get update
+apt-get -yq install libmpdec-dev

.github/workflows/reusable-docs.yml

@@ -66,7 +66,7 @@ jobs:
       run: |
         set -Eeuo pipefail
         # Build docs with the nit-picky option; write warnings to file
-        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html
     - name: 'Check warnings'
       if: github.event_name == 'pull_request'
       run: |
@@ -75,6 +75,18 @@ jobs:
           --fail-if-regression \
           --fail-if-improved \
           --fail-if-new-news-nit
+    - name: 'Build EPUB documentation'
+      continue-on-error: true
+      run: |
+        set -Eeuo pipefail
+        make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet" epub
+        pip install epubcheck
+        epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt
+    - name: 'Check for fatal errors in EPUB'
+      if: github.event_name == 'pull_request'
+      continue-on-error: true  # until gh-136155 is fixed
+      run: |
+        python Doc/tools/check-epub.py
 
   # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release
   doctest:

.pre-commit-config.yaml

@@ -34,6 +34,13 @@ repos:
         name: Run Black on Tools/jit/
         files: ^Tools/jit/
 
+  - repo: https://github.com/Lucas-C/pre-commit-hooks
+    rev: v1.5.5
+    hooks:
+      - id: remove-tabs
+        types: [python]
+        exclude: ^Tools/c-analyzer/cpython/_parser.py
+
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:

Android/android.py

@@ -175,7 +175,7 @@ def unpack_deps(host, prefix_dir):
     os.chdir(prefix_dir)
     deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download"
     for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4",
-                     "sqlite-3.49.1-0", "xz-5.4.6-1"]:
+                     "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]:
         filename = f"{name_ver}-{host}.tar.gz"
         download(f"{deps_url}/{name_ver}/{filename}")
         shutil.unpack_archive(filename)

Doc/c-api/arg.rst

@@ -241,9 +241,11 @@ the Python object to the required type.
 
 For signed integer formats, :exc:`OverflowError` is raised if the value
 is out of range for the C type.
-For unsigned integer formats, no range checking is done --- the
+For unsigned integer formats, the
 most significant bits are silently truncated when the receiving field is too
-small to receive the value.
+small to receive the value, and :exc:`DeprecationWarning` is emitted when
+the value is larger than the maximal value for the C type or less than
+the minimal value for the corresponding signed integer type of the same size.
 
 ``b`` (:class:`int`) [unsigned char]
    Convert a nonnegative Python integer to an unsigned tiny integer, stored in a C
@@ -252,27 +254,25 @@ small to receive the value.
 ``B`` (:class:`int`) [unsigned char]
    Convert a Python integer to a tiny integer without overflow checking, stored in a C
    :c:expr:`unsigned char`.
+   Convert a Python integer to a C :c:expr:`unsigned char`.
 
 ``h`` (:class:`int`) [short int]
    Convert a Python integer to a C :c:expr:`short int`.
 
 ``H`` (:class:`int`) [unsigned short int]
-   Convert a Python integer to a C :c:expr:`unsigned short int`, without overflow
-   checking.
+   Convert a Python integer to a C :c:expr:`unsigned short int`.
 
 ``i`` (:class:`int`) [int]
    Convert a Python integer to a plain C :c:expr:`int`.
 
 ``I`` (:class:`int`) [unsigned int]
-   Convert a Python integer to a C :c:expr:`unsigned int`, without overflow
-   checking.
+   Convert a Python integer to a C :c:expr:`unsigned int`.
 
 ``l`` (:class:`int`) [long int]
    Convert a Python integer to a C :c:expr:`long int`.
 
 ``k`` (:class:`int`) [unsigned long]
-   Convert a Python integer to a C :c:expr:`unsigned long` without
-   overflow checking.
+   Convert a Python integer to a C :c:expr:`unsigned long`.
 
    .. versionchanged:: 3.14
       Use :meth:`~object.__index__` if available.
@@ -281,8 +281,7 @@ small to receive the value.
    Convert a Python integer to a C :c:expr:`long long`.
 
 ``K`` (:class:`int`) [unsigned long long]
-   Convert a Python integer to a C :c:expr:`unsigned long long`
-   without overflow checking.
+   Convert a Python integer to a C :c:expr:`unsigned long long`.
 
    .. versionchanged:: 3.14
       Use :meth:`~object.__index__` if available.
@@ -310,6 +309,14 @@ small to receive the value.
 ``D`` (:class:`complex`) [Py_complex]
    Convert a Python complex number to a C :c:type:`Py_complex` structure.
 
+.. deprecated:: next
+
+   For unsigned integer formats ``B``, ``H``, ``I``, ``k`` and ``K``,
+   :exc:`DeprecationWarning` is emitted when the value is larger than
+   the maximal value for the C type or less than the minimal value for
+   the corresponding signed integer type of the same size.
+
+
 Other objects
 -------------
 
@@ -686,6 +693,12 @@ Building values
    ``p`` (:class:`bool`) [int]
       Convert a C :c:expr:`int` to a Python :class:`bool` object.
 
+      Be aware that this format requires an ``int`` argument.
+      Unlike most other contexts in C, variadic arguments are not coerced to
+      a suitable type automatically.
+      You can convert another type (for example, a pointer or a float) to a
+      suitable ``int`` value using ``(x) ? 1 : 0`` or ``!!x``.
+
       .. versionadded:: 3.14
 
    ``c`` (:class:`bytes` of length 1) [char]

Doc/c-api/capsule.rst

@@ -105,9 +105,19 @@ Refer to :ref:`using-capsules` for more information on using these objects.
    ``module.attribute``.  The *name* stored in the capsule must match this
    string exactly.
 
+   This function splits *name* on the ``.`` character, and imports the first
+   element. It then processes further elements using attribute lookups.
+
    Return the capsule's internal *pointer* on success.  On failure, set an
    exception and return ``NULL``.
 
+   .. note::
+
+      If *name* points to an attribute of some submodule or subpackage, this
+      submodule or subpackage must be previously imported using other means
+      (for example, by using :c:func:`PyImport_ImportModule`) for the
+      attribute lookups to succeed.
+
    .. versionchanged:: 3.3
       *no_block* has no effect anymore.