Merge branch 'main' into implement-119127-again

Author: rhettinger

.devcontainer/Dockerfile

@@ -2,7 +2,7 @@ FROM docker.io/library/fedora:40
 
 ENV CC=clang
 
-ENV WASI_SDK_VERSION=22
+ENV WASI_SDK_VERSION=24
 ENV WASI_SDK_PATH=/opt/wasi-sdk
 
 ENV WASMTIME_HOME=/opt/wasmtime
@@ -14,7 +14,7 @@ RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,cla
     dnf -y clean all
 
 RUN mkdir ${WASI_SDK_PATH} && \
-    curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-linux.tar.gz | \
+    curl --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | \
     tar --strip-components 1 --directory ${WASI_SDK_PATH} --extract --gunzip
 
 RUN mkdir --parents ${WASMTIME_HOME} && \

.gitattributes

@@ -27,8 +27,6 @@ Lib/test/cjkencodings/*                    noeol
 Lib/test/tokenizedata/coding20731.py       noeol
 Lib/test/decimaltestdata/*.decTest         noeol
 Lib/test/test_email/data/*.txt             noeol
-Lib/test/test_importlib/resources/data01/*           noeol
-Lib/test/test_importlib/resources/namespacedata01/*  noeol
 Lib/test/xmltestdata/*                     noeol
 
 # Shell scripts should have LF even on Windows because of Cygwin

.github/CODEOWNERS

@@ -33,11 +33,11 @@ Objects/frameobject.c         @markshannon
 Objects/call.c                @markshannon
 Python/ceval*.c               @markshannon
 Python/ceval*.h               @markshannon
+Python/codegen.c              @markshannon @iritkatriel
 Python/compile.c              @markshannon @iritkatriel
 Python/assemble.c             @markshannon @iritkatriel
 Python/flowgraph.c            @markshannon @iritkatriel
 Python/instruction_sequence.c @iritkatriel
-Python/ast_opt.c              @isidentical
 Python/bytecodes.c            @markshannon
 Python/optimizer*.c           @markshannon
 Python/optimizer_analysis.c   @Fidget-Spinner
@@ -157,10 +157,12 @@ Include/internal/pycore_time.h  @pganssle @abalkin
 /Tools/cases_generator/        @markshannon
 
 # AST
-Python/ast.c                  @isidentical @JelleZijlstra
-Parser/asdl.py                @isidentical @JelleZijlstra
-Parser/asdl_c.py              @isidentical @JelleZijlstra
-Lib/ast.py                    @isidentical @JelleZijlstra
+Python/ast.c                  @isidentical @JelleZijlstra @eclips4
+Python/ast_opt.c              @isidentical @eclips4
+Parser/asdl.py                @isidentical @JelleZijlstra @eclips4
+Parser/asdl_c.py              @isidentical @JelleZijlstra @eclips4
+Lib/ast.py                    @isidentical @JelleZijlstra @eclips4
+Lib/test/test_ast/            @eclips4
 
 # Mock
 /Lib/unittest/mock.py         @cjw296
@@ -211,8 +213,9 @@ Doc/c-api/stable.rst          @encukou
 
 **/*ensurepip*                @pfmoore @pradyunsg
 
-**/*idlelib*                  @terryjreedy
 /Doc/library/idle.rst         @terryjreedy
+**/*idlelib*                  @terryjreedy
+**/*turtledemo*               @terryjreedy
 
 **/*annotationlib*            @JelleZijlstra
 **/*typing*                   @JelleZijlstra @AlexWaygood
@@ -275,3 +278,6 @@ Lib/test/test_interpreters/   @ericsnowcurrently
 # Config Parser
 Lib/configparser.py           @jaraco
 Lib/test/test_configparser.py @jaraco
+
+# Doc sections
+Doc/reference/                @willingc

.github/workflows/build.yml

@@ -195,14 +195,15 @@ jobs:
 
   build_ubuntu_ssltests:
     name: 'Ubuntu SSL tests with OpenSSL'
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
     timeout-minutes: 60
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     strategy:
       fail-fast: false
       matrix:
-        openssl_ver: [1.1.1w, 3.0.13, 3.1.5, 3.2.1]
+        os: [ubuntu-22.04]
+        openssl_ver: [3.0.15, 3.1.7, 3.2.3, 3.3.2]
     env:
       OPENSSL_VER: ${{ matrix.openssl_ver }}
       MULTISSL_DIR: ${{ github.workspace }}/multissl
@@ -231,7 +232,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -266,7 +267,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_hypothesis == 'true'
     env:
-      OPENSSL_VER: 3.0.13
+      OPENSSL_VER: 3.0.15
       PYTHONSTRICTEXTENSIONBUILD: 1
     steps:
     - uses: actions/checkout@v4
@@ -380,7 +381,7 @@ jobs:
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
     env:
-      OPENSSL_VER: 3.0.13
+      OPENSSL_VER: 3.0.15
       PYTHONSTRICTEXTENSIONBUILD: 1
       ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0
     steps:
@@ -410,7 +411,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux

.github/workflows/jit.yml

@@ -110,17 +110,15 @@ jobs:
       - name: Native Windows
         if: runner.os == 'Windows' && matrix.architecture != 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '--pgo' }} -p ${{ matrix.architecture }}
           ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 
       # No PGO or tests (yet):
       - name: Emulated Windows
         if: runner.os == 'Windows' && matrix.architecture == 'ARM64'
         run: |
-          choco upgrade llvm -y
-          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}
+          choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0
           ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }}
 
       - name: Native macOS
@@ -159,7 +157,7 @@ jobs:
           CC="${{ matrix.compiler == 'clang' && 'clang --target=$HOST' || '$HOST-gcc' }}" \
             CPP="$CC --preprocess" \
             HOSTRUNNER=qemu-${{ matrix.architecture }} \
-            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--enable-optimizations --with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
+            ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '--with-lto' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes
           make all --jobs 4
           ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3
 

.github/workflows/reusable-change-detection.yml

@@ -126,13 +126,18 @@ jobs:
           .github/workflows/reusable-docs.yml
         format: csv  # works for paths with spaces
     - name: Check for docs changes
+      # We only want to run this on PRs when related files are changed,
+      # or when user triggers manual workflow run.
       if: >-
-        github.event_name == 'pull_request'
-        && steps.changed-docs-files.outputs.added_modified_renamed != ''
+        (
+          github.event_name == 'pull_request'
+          && steps.changed-docs-files.outputs.added_modified_renamed != ''
+        ) || github.event_name == 'workflow_dispatch'
       id: docs-changes
       run: |
         echo "run-docs=true" >> "${GITHUB_OUTPUT}"
     - name: Get a list of the MSI installer-related files
+      if: github.event_name == 'pull_request'
       id: changed-win-msi-files
       uses: Ana06/[email protected]
       with:
@@ -141,10 +146,13 @@ jobs:
           .github/workflows/reusable-windows-msi.yml
         format: csv  # works for paths with spaces
     - name: Check for changes in MSI installer-related files
+      # We only want to run this on PRs when related files are changed,
+      # or when user triggers manual workflow run.
       if: >-
-        steps.changed-win-msi-files.outputs.added_modified_renamed != ''
+        (
+          github.event_name == 'pull_request'
+          && steps.changed-win-msi-files.outputs.added_modified_renamed != ''
+        ) || github.event_name == 'workflow_dispatch'
       id: win-msi-changes
       run: |
         echo "run-win-msi=true" >> "${GITHUB_OUTPUT}"
-
-...

.github/workflows/reusable-docs.yml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+env:
+  FORCE_COLOR: 1
+
 jobs:
   build_doc:
     name: 'Docs'
@@ -25,9 +28,15 @@ jobs:
     - name: 'Check out latest PR branch commit'
       uses: actions/checkout@v4
       with:
-        ref: ${{ github.event.pull_request.head.sha }}
+        ref: >-
+          ${{
+            github.event_name == 'pull_request'
+            && github.event.pull_request.head.sha
+            || ''
+          }}
     # Adapted from https://github.com/actions/checkout/issues/520#issuecomment-1167205721
     - name: 'Fetch commits to get branch diff'
+      if: github.event_name == 'pull_request'
       run: |
         # Fetch enough history to find a common ancestor commit (aka merge-base):
         git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \

.github/workflows/reusable-macos.yml

@@ -35,7 +35,7 @@ jobs:
         path: config.cache
         key: ${{ github.job }}-${{ inputs.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}
     - name: Install Homebrew dependencies
-      run: brew install pkg-config [email protected] xz gdbm tcl-tk
+      run: brew install pkg-config [email protected] xz gdbm tcl-tk make
     - name: Configure CPython
       run: |
         GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \
@@ -44,14 +44,27 @@ jobs:
           --config-cache \
           --with-pydebug \
           --enable-slower-safety \
+          --enable-safety \
           ${{ inputs.free-threading && '--disable-gil' || '' }} \
           --prefix=/opt/python-dev \
           --with-openssl="$(brew --prefix [email protected])"
     - name: Build CPython
-      run: set -o pipefail; make -j8 2>&1 | tee compiler_output.txt
+      if : ${{ inputs.free-threading || inputs.os != 'macos-13' }}
+      run: gmake -j8
+    - name: Build CPython for compiler warning check
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt
     - name: Display build info
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output.txt --warning-ignore-file-path=Tools/build/.warningignore_macos --compiler-output-type=clang
+      if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }}
+      run: >-
+        python3 Tools/build/check_warnings.py
+        --compiler-output-file-path=compiler_output_macos.txt
+        --warning-ignore-file-path=Tools/build/.warningignore_macos
+        --compiler-output-type=clang
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="./"
     - name: Tests
       run: make test

.github/workflows/reusable-ubuntu.yml

@@ -14,10 +14,14 @@ jobs:
   build_ubuntu_reusable:
     name: 'build and test'
     timeout-minutes: 60
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04]
     env:
       FORCE_COLOR: 1
-      OPENSSL_VER: 3.0.13
+      OPENSSL_VER: 3.0.15
       PYTHONSTRICTEXTENSIONBUILD: 1
       TERM: linux
     steps:
@@ -36,7 +40,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: ./multissl/openssl/${{ env.OPENSSL_VER }}
-        key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
+        key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }}
     - name: Install OpenSSL
       if: steps.cache-openssl.outputs.cache-hit != 'true'
       run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux
@@ -67,20 +71,33 @@ jobs:
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: >-
         ../cpython-ro-srcdir/configure
-        CFLAGS="-fdiagnostics-format=json"
         --config-cache
         --with-pydebug
         --enable-slower-safety
+        --enable-safety
         --with-openssl=$OPENSSL_DIR
         ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }}
     - name: Build CPython out-of-tree
+      if: ${{ inputs.free-threading }}
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
-      run: set -o pipefail; make -j4 2>&1 | tee compiler_output.txt
+      run: make -j4
+    - name: Build CPython out-of-tree (for compiler warning check)
+      if: ${{ !inputs.free-threading}}
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: set -o pipefail; make -j4 --output-sync 2>&1 | tee compiler_output_ubuntu.txt
     - name: Display build info
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
     - name: Check compiler warnings
-      run: python Tools/build/check_warnings.py --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output.txt --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu --compiler-output-type=json
+      if: ${{ !inputs.free-threading }}
+      run: >-
+        python Tools/build/check_warnings.py
+        --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt
+        --warning-ignore-file-path ${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu
+        --compiler-output-type=gcc
+        --fail-on-regression
+        --fail-on-improvement
+        --path-prefix="../cpython-ro-srcdir/"
     - name: Remount sources writable for tests
       # some tests write to srcdir, lack of pyc files slows down testing
       run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw

.github/workflows/reusable-wasi.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-22.04
     env:
       WASMTIME_VERSION: 22.0.0
-      WASI_SDK_VERSION: 22
+      WASI_SDK_VERSION: 24
       WASI_SDK_PATH: /opt/wasi-sdk
       CROSS_BUILD_PYTHON: cross-build/build
       CROSS_BUILD_WASI: cross-build/wasm32-wasi
@@ -33,7 +33,7 @@ jobs:
       if: steps.cache-wasi-sdk.outputs.cache-hit != 'true'
       run: |
         mkdir ${{ env.WASI_SDK_PATH }} && \
-        curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-linux.tar.gz | \
+        curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-x86_64-linux.tar.gz | \
         tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip
     - name: "Configure ccache action"
       uses: hendrikmuhs/[email protected]