From 7504fe61b843b6e637138903cf7f1290f1c665fe Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Wed, 7 May 2025 18:45:45 +0300 Subject: [PATCH 001/713] Post 3.14.0b1 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c9dd659046a367..03ac5843b62066 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,7 +24,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.14.0b1" +#define PY_VERSION "3.14.0b1+" /*--end constants--*/ From 0eb2291a7e85062dba387dfecaee94858db8a0a9 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 7 May 2025 16:35:46 -0500 Subject: [PATCH 002/713] [3.14] Forward-port 'check-abi' CI job from 3.13 (GH-133614) Also add the python3.14.abi file as generated by the new job and remove the 'main branch only' entry from .gitignore. The only difference from the 3.13 job is the addition of `with.python-version: 3.x` to the `setup-python` configuration to pacify a warning. --- .github/workflows/build.yml | 47 + .gitignore | 3 - Doc/data/python3.14.abi | 31341 ++++++++++++++++++++++++++++++++++ 3 files changed, 31388 insertions(+), 3 deletions(-) create mode 100644 Doc/data/python3.14.abi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b192508c78685c..3478af18f0e9cb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -43,6 +43,53 @@ jobs: if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check-abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-22.04 # 24.04 causes spurious errors + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + id: check + run: | + if ! make check-abidump; then + echo "Generated ABI file is not up to date." + echo "Please add the release manager of this branch as a reviewer of this PR." + echo "" + echo "The up to date ABI file should be attached to this build as an artifact." + echo "" + echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump" + echo "" + exit 1 + fi + - name: Generate updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + run: | + make regen-abidump + - uses: actions/upload-artifact@v4 + name: Publish updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + with: + name: abi-data + path: ./Doc/data/*.abi + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job diff --git a/.gitignore b/.gitignore index 2a6f249275c32e..7965f6a404368e 100644 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,3 @@ Python/frozen_modules/MANIFEST # Ignore ./python binary on Unix but still look into ./Python/ directory. /python !/Python/ - -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi new file mode 100644 index 00000000000000..c13b44e0945389 --- /dev/null +++ b/Doc/data/python3.14.abi @@ -0,0 +1,31341 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From e2d440ebc6d2457d23c80da75a5b1b40b17f0b4e Mon Sep 17 00:00:00 2001 From: Brian Schubert Date: Wed, 7 May 2025 18:03:30 -0400 Subject: [PATCH 003/713] [3.14] gh-131535: Fix stale example in html.parser docs, make examples doctests (GH-131551) (GH-133589) (cherry picked from commit ee76e36d76a0e6916c0afc41228b043ab5174685) --- Doc/library/html.parser.rst | 51 +++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 6d433b5a04fc4a..dd67fc34e856f1 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -43,7 +43,9 @@ Example HTML Parser Application As a basic example, below is a simple HTML parser that uses the :class:`HTMLParser` class to print out start tags, end tags, and data -as they are encountered:: +as they are encountered: + +.. testcode:: from html.parser import HTMLParser @@ -63,7 +65,7 @@ as they are encountered:: The output will then be: -.. code-block:: none +.. testoutput:: Encountered a start tag: html Encountered a start tag: head @@ -230,7 +232,9 @@ Examples -------- The following class implements a parser that will be used to illustrate more -examples:: +examples: + +.. testcode:: from html.parser import HTMLParser from html.entities import name2codepoint @@ -266,13 +270,17 @@ examples:: parser = MyHTMLParser() -Parsing a doctype:: +Parsing a doctype: + +.. doctest:: >>> parser.feed('') Decl : DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" -Parsing an element with a few attributes and a title:: +Parsing an element with a few attributes and a title: + +.. doctest:: >>> parser.feed('The Python logo') Start tag: img @@ -285,7 +293,9 @@ Parsing an element with a few attributes and a title:: End tag : h1 The content of ``script`` and ``style`` elements is returned as is, without -further parsing:: +further parsing: + +.. doctest:: >>> parser.feed('') Start tag: style @@ -300,16 +310,25 @@ further parsing:: Data : alert("hello!"); End tag : script -Parsing comments:: +Parsing comments: + +.. doctest:: - >>> parser.feed('' + >>> parser.feed('' ... '') - Comment : a comment + Comment : a comment Comment : [if IE 9]>IE-specific content'``):: +correct char (note: these 3 references are all equivalent to ``'>'``): +.. doctest:: + + >>> parser = MyHTMLParser() + >>> parser.feed('>>>') + Data : >>> + + >>> parser = MyHTMLParser(convert_charrefs=False) >>> parser.feed('>>>') Named ent: > Num ent : > @@ -317,18 +336,22 @@ correct char (note: these 3 references are all equivalent to ``'>'``):: Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but :meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: +(unless *convert_charrefs* is set to ``True``): - >>> for chunk in ['buff', 'ered ', 'text']: +.. doctest:: + + >>> for chunk in ['buff', 'ered', ' text']: ... parser.feed(chunk) ... Start tag: span Data : buff Data : ered - Data : text + Data : text End tag : span -Parsing invalid HTML (e.g. unquoted attributes) also works:: +Parsing invalid HTML (e.g. unquoted attributes) also works: + +.. doctest:: >>> parser.feed('

tag soup

') Start tag: p From cc588aaa5f9cd0e412fe6cc434f74c0039937418 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 00:10:51 +0200 Subject: [PATCH 004/713] [3.14] gh-133361: move the explanation of dict equal before its use (GH-133424) (#133620) gh-133361: move the explanation of dict equal before its use (GH-133424) Also move up the explanation of insertion order preservation. Both paragraphs seemed out of place down where they were. --------- (cherry picked from commit 61ac88c06e0286cfc6465a47552d41b7dd7e083e) Co-authored-by: Yongzi Li <1538321957@qq.com> Co-authored-by: Terry Jan Reedy --- Doc/library/stdtypes.rst | 55 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 39aaa5da0786f8..61d39a6671caed 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4823,7 +4823,13 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal to + Providing keyword arguments as in the first example only works for keys that + are valid Python identifiers. Otherwise, any valid keys can be used. + + Dictionaries compare equal if and only if they have the same ``(key, + value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise + :exc:`TypeError`. To illustrate dictionary creation and equality, + the following examples all return a dictionary equal to ``{"one": 1, "two": 2, "three": 3}``:: >>> a = dict(one=1, two=2, three=3) @@ -4838,6 +4844,27 @@ can be used interchangeably to index the same dictionary entry. Providing keyword arguments as in the first example only works for keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key does not + affect the order. Keys added after deletion are inserted at the end. :: + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + .. versionchanged:: 3.7 + Dictionary order is guaranteed to be insertion order. This behavior was + an implementation detail of CPython from 3.6. These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -5008,32 +5035,6 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.9 - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - - Dictionaries preserve insertion order. Note that updating a key does not - affect the order. Keys added after deletion are inserted at the end. :: - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - .. versionchanged:: 3.7 - Dictionary order is guaranteed to be insertion order. This behavior was - an implementation detail of CPython from 3.6. - Dictionaries and dictionary views are reversible. :: >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} From c98b5a9369e863d56cf4f919445ca6d743369530 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 00:20:17 +0200 Subject: [PATCH 005/713] [3.14] gh-133558: Skip test_pdb tests on FreeBSD (GH-133566) (#133616) gh-133558: Skip test_pdb tests on FreeBSD (GH-133566) '\x08' is not interpreted as backspace on FreeBSD. (cherry picked from commit 79b8a32fcb30b2c6549bacecdc7081ebd50d6e17) Co-authored-by: Victor Stinner --- Lib/test/test_pdb.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 54797d7898ff33..68c2b508fa64dc 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -4848,6 +4848,8 @@ def f(): self.assertIn(b'I love Python', output) + @unittest.skipIf(sys.platform.startswith('freebsd'), + '\\x08 is not interpreted as backspace on FreeBSD') def test_multiline_auto_indent(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() @@ -4886,6 +4888,8 @@ def test_multiline_completion(self): self.assertIn(b'42', output) + @unittest.skipIf(sys.platform.startswith('freebsd'), + '\\x08 is not interpreted as backspace on FreeBSD') def test_multiline_indent_completion(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() From 54c3aa1597ef3b7a967a27a5ea8577cefeb59769 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 00:26:06 +0200 Subject: [PATCH 006/713] [3.14] gh-133516: Raise `ValueError` when constants `True`, `False` or `None` are used as an identifier after NFKC normalization (GH-133523) (#133596) --- Lib/test/test_ast/test_ast.py | 11 +++++++++++ ...2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst | 2 ++ Parser/pegen.c | 15 +++++++++++++++ 3 files changed, 28 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 09cf3186e05cc1..02628868db008c 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -821,6 +821,17 @@ def test_constant_as_name(self): with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") + def test_constant_as_unicode_name(self): + constants = [ + ("True", b"Tru\xe1\xb5\x89"), + ("False", b"Fal\xc5\xbfe"), + ("None", b"N\xc2\xbane"), + ] + for constant in constants: + with self.assertRaisesRegex(ValueError, + f"identifier field can't represent '{constant[0]}' constant"): + ast.parse(constant[1], mode="eval") + def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst new file mode 100644 index 00000000000000..b93ba11f93252b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst @@ -0,0 +1,2 @@ +Raise :exc:`ValueError` when constants ``True``, ``False`` or ``None`` are +used as an identifier after NFKC normalization. diff --git a/Parser/pegen.c b/Parser/pegen.c index 3efeba78450d1a..81aad47010181c 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -549,6 +549,21 @@ _PyPegen_new_identifier(Parser *p, const char *n) } id = id2; } + static const char * const forbidden[] = { + "None", + "True", + "False", + NULL + }; + for (int i = 0; forbidden[i] != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(id, forbidden[i])) { + PyErr_Format(PyExc_ValueError, + "identifier field can't represent '%s' constant", + forbidden[i]); + Py_DECREF(id); + goto error; + } + } PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternImmortal(interp, &id); if (_PyArena_AddPyObject(p->arena, id) < 0) From c39bc81b70cdd67b72a3c12cf2c6418414e266c9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 May 2025 18:00:33 -0600 Subject: [PATCH 007/713] [3.14] gh-132775: Unrevert "Add _PyCode_VerifyStateless()" (gh-133625) This reverts commit 3c73cf5 (gh-133497), which itself reverted the original commit d270bb5 (gh-133221). We reverted the original change due to failing android tests. The checks in _PyCode_CheckNoInternalState() were too strict, so we've relaxed them. --- Include/internal/pycore_code.h | 41 ++++++ Include/internal/pycore_function.h | 7 ++ Include/internal/pycore_opcode_utils.h | 2 + Lib/test/_code_definitions.py | 26 ++++ Lib/test/test_code.py | 54 +++++--- Modules/_testinternalcapi.c | 43 +++++++ Objects/codeobject.c | 168 ++++++++++++++++++++++--- Objects/funcobject.c | 54 ++++++++ 8 files changed, 358 insertions(+), 37 deletions(-) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b135e30b7ad62a..37a747aa4e3e46 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -614,6 +614,47 @@ PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts( PyObject *globalsns, PyObject *builtinsns); + +/* "Stateless" code is a function or code object which does not rely on + * external state or internal state. It may rely on arguments and + * builtins, but not globals or a closure. Thus it does not rely + * on __globals__ or __closure__, and a stateless function + * is equivalent to its code object. + * + * Stateless code also does not keep any persistent state + * of its own, so it can't have any executors, monitoring, + * instrumentation, or "extras" (i.e. co_extra). + * + * Stateless code may create nested functions, including closures. + * However, nested functions must themselves be stateless, except they + * *can* close on the enclosing locals. + * + * Stateless code may return any value, including nested functions and closures. + * + * Stateless code that takes no arguments and doesn't return anything + * may be treated like a script. + * + * We consider stateless code to be "portable" if it does not return + * any object that holds a reference to any of the code's locals. Thus + * generators and coroutines are not portable. Likewise a function + * that returns a closure is not portable. The concept of + * portability is useful in cases where the code is run + * in a different execution context than where + * the return value will be used. */ + +PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **); +PyAPI_FUNC(int) _PyCode_CheckNoExternalState( + PyCodeObject *, + _PyCode_var_counts_t *, + const char **); +PyAPI_FUNC(int) _PyCode_VerifyStateless( + PyThreadState *, + PyCodeObject *, + PyObject *globalnames, + PyObject *globalsns, + PyObject *builtinsns); + +PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 209252b2ddc0de..a30d52d49bdc4d 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -35,6 +35,13 @@ PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_cod extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); + +/* See pycore_code.h for explanation about what "stateless" means. */ + +PyAPI_FUNC(int) +_PyFunction_VerifyStateless(PyThreadState *, PyObject *); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 62af06dc01c125..79a1a242556a52 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -56,6 +56,8 @@ extern "C" { #define IS_RETURN_OPCODE(opcode) \ (opcode == RETURN_VALUE) +#define IS_RAISE_OPCODE(opcode) \ + (opcode == RAISE_VARARGS || opcode == RERAISE) /* Flags used in the oparg for MAKE_FUNCTION */ diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index c3daa0dccf5df3..d64ac45d85f396 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -178,6 +178,32 @@ def ham_C_closure(z): *NESTED_FUNCTIONS, ] +STATELESS_FUNCTIONS = [ + spam, + spam_minimal, + spam_with_builtins, + spam_args_attrs_and_builtins, + spam_returns_arg, + spam_annotated, + spam_with_inner_not_closure, + spam_with_inner_closure, + spam_N, + spam_C, + spam_NN, + spam_NC, + spam_CN, + spam_CC, + eggs_nested, + eggs_nested_N, + ham_nested, + ham_C_nested +] +STATELESS_CODE = [ + *STATELESS_FUNCTIONS, + spam_with_globals_and_builtins, + spam_full, +] + # generators diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index b646042a3b84b0..6715ee051336a1 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -220,6 +220,7 @@ import _testinternalcapi except ModuleNotFoundError: _testinternalcapi = None +import test._code_definitions as defs COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -671,7 +672,6 @@ def test_local_kinds(self): VARARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_POS VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW - import test._code_definitions as defs funcs = { defs.spam_minimal: {}, defs.spam_with_builtins: { @@ -897,7 +897,6 @@ def new_var_counts(*, }, } - import test._code_definitions as defs funcs = { defs.spam_minimal: new_var_counts(), defs.spam_with_builtins: new_var_counts( @@ -1025,42 +1024,35 @@ def new_var_counts(*, counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) - def func_with_globals_and_builtins(): - mod1 = _testinternalcapi - mod2 = dis - mods = (mod1, mod2) - checks = tuple(callable(m) for m in mods) - return callable(mod2), tuple(mods), list(mods), checks - - func = func_with_globals_and_builtins + func = defs.spam_with_globals_and_builtins with self.subTest(f'{func} code'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) with self.subTest(f'{func} with own globals and builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 3), + purelocals=5, + globalvars=(2, 4), ) counts = _testinternalcapi.get_code_var_counts(func) self.assertEqual(counts, expected) with self.subTest(f'{func} without globals'): expected = new_var_counts( - purelocals=4, - globalvars=(0, 3, 2), + purelocals=5, + globalvars=(0, 4, 2), ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}) self.assertEqual(counts, expected) with self.subTest(f'{func} without both'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}, builtinsns={}) @@ -1068,12 +1060,34 @@ def func_with_globals_and_builtins(): with self.subTest(f'{func} without builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 0, 3), + purelocals=5, + globalvars=(2, 0, 4), ) counts = _testinternalcapi.get_code_var_counts(func, builtinsns={}) self.assertEqual(counts, expected) + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_stateless(self): + self.maxDiff = None + + for func in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + _testinternalcapi.verify_stateless_code(func.__code__) + for func in defs.STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + _testinternalcapi.verify_stateless_code(func) + + for func in defs.FUNCTIONS: + if func not in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func.__code__) + + if func not in defs.STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ae060c95fd5a01..63f1d079d8d312 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1165,6 +1165,47 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) return NULL; } +static PyObject * +verify_stateless_code(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *codearg; + PyObject *globalnames = NULL; + PyObject *globalsns = NULL; + PyObject *builtinsns = NULL; + static char *kwlist[] = {"code", "globalnames", + "globalsns", "builtinsns", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O|O!O!O!:get_code_var_counts", kwlist, + &codearg, &PySet_Type, &globalnames, + &PyDict_Type, &globalsns, &PyDict_Type, &builtinsns)) + { + return NULL; + } + if (PyFunction_Check(codearg)) { + if (globalsns == NULL) { + globalsns = PyFunction_GET_GLOBALS(codearg); + } + if (builtinsns == NULL) { + builtinsns = PyFunction_GET_BUILTINS(codearg); + } + codearg = PyFunction_GET_CODE(codearg); + } + else if (!PyCode_Check(codearg)) { + PyErr_SetString(PyExc_TypeError, + "argument must be a code object or a function"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)codearg; + + if (_PyCode_VerifyStateless( + tstate, code, globalnames, globalsns, builtinsns) < 0) + { + return NULL; + } + Py_RETURN_NONE; +} + #ifdef _Py_TIER2 static PyObject * @@ -2292,6 +2333,8 @@ static PyMethodDef module_functions[] = { {"get_co_localskinds", get_co_localskinds, METH_O, NULL}, {"get_code_var_counts", _PyCFunction_CAST(get_code_var_counts), METH_VARARGS | METH_KEYWORDS, NULL}, + {"verify_stateless_code", _PyCFunction_CAST(verify_stateless_code), + METH_VARARGS | METH_KEYWORDS, NULL}, #ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 92eaf5e00bc7dd..4f06a36a130207 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1955,12 +1955,130 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, } +int +_PyCode_CheckNoInternalState(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + // We don't worry about co_executors, co_instrumentation, + // or co_monitoring. They are essentially ephemeral. + if (co->co_extra != NULL) { + errmsg = "only basic code objects are supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_CheckNoExternalState(PyCodeObject *co, _PyCode_var_counts_t *counts, + const char **p_errmsg) +{ + const char *errmsg = NULL; + assert(counts->locals.hidden.total == 0); + if (counts->numfree > 0) { // It's a closure. + errmsg = "closures not supported"; + } + else if (counts->unbound.globals.numglobal > 0) { + errmsg = "globals not supported"; + } + else if (counts->unbound.globals.numbuiltin > 0 + && counts->unbound.globals.numunknown > 0) + { + errmsg = "globals not supported"; + } + // Otherwise we don't check counts.unbound.globals.numunknown since we can't + // distinguish beween globals and builtins here. + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_VerifyStateless(PyThreadState *tstate, + PyCodeObject *co, PyObject *globalnames, + PyObject *globalsns, PyObject *builtinsns) +{ + const char *errmsg; + _PyCode_var_counts_t counts = {0}; + _PyCode_GetVarCounts(co, &counts); + if (_PyCode_SetUnboundVarCounts( + tstate, co, &counts, globalnames, NULL, + globalsns, builtinsns) < 0) + { + return -1; + } + // We may consider relaxing the internal state constraints + // if it becomes a problem. + if (!_PyCode_CheckNoInternalState(co, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + if (builtinsns != NULL) { + // Make sure the next check will fail for globals, + // even if there aren't any builtins. + counts.unbound.globals.numbuiltin += 1; + } + if (!_PyCode_CheckNoExternalState(co, &counts, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + // Note that we don't check co->co_flags & CO_NESTED for anything here. + return 0; +} + + +int +_PyCode_CheckPureFunction(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + if (co->co_flags & CO_GENERATOR) { + errmsg = "generators not supported"; + } + else if (co->co_flags & CO_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ITERABLE_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ASYNC_GENERATOR) { + errmsg = "generators not supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + /* Here "value" means a non-None value, since a bare return is identical * to returning None explicitly. Likewise a missing return statement * at the end of the function is turned into "return None". */ static int code_returns_only_none(PyCodeObject *co) { + if (!_PyCode_CheckPureFunction(co, NULL)) { + return 0; + } + int len = (int)Py_SIZE(co); + assert(len > 0); + + // The last instruction either returns or raises. We can take advantage + // of that for a quick exit. + _Py_CODEUNIT final = _Py_GetBaseCodeUnit(co, len-1); + // Look up None in co_consts. Py_ssize_t nconsts = PyTuple_Size(co->co_consts); int none_index = 0; @@ -1971,26 +2089,42 @@ code_returns_only_none(PyCodeObject *co) } if (none_index == nconsts) { // None wasn't there, which means there was no implicit return, - // "return", or "return None". That means there must be - // an explicit return (non-None). - return 0; - } + // "return", or "return None". - // Walk the bytecode, looking for RETURN_VALUE. - Py_ssize_t len = Py_SIZE(co); - for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { - _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); - if (IS_RETURN_OPCODE(inst.op.code)) { - assert(i != 0); - // Ignore it if it returns None. - _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); - if (prev.op.code == LOAD_CONST) { - // We don't worry about EXTENDED_ARG for now. - if (prev.op.arg == none_index) { - continue; + // That means there must be + // an explicit return (non-None), or it only raises. + if (IS_RETURN_OPCODE(final.op.code)) { + // It was an explicit return (non-None). + return 0; + } + // It must end with a raise then. We still have to walk the + // bytecode to see if there's any explicit return (non-None). + assert(IS_RAISE_OPCODE(final.op.code)); + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + // We alraedy know it isn't returning None. + return 0; + } + } + // It must only raise. + } + else { + // Walk the bytecode, looking for RETURN_VALUE. + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + assert(i != 0); + // Ignore it if it returns None. + _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); + if (prev.op.code == LOAD_CONST) { + // We don't worry about EXTENDED_ARG for now. + if (prev.op.arg == none_index) { + continue; + } } + return 0; } - return 0; } } return 1; diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 56df5730db0c55..27214a129c2fb8 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1,12 +1,14 @@ /* Function object implementation */ #include "Python.h" +#include "pycore_code.h" // _PyCode_VerifyStateless() #include "pycore_dict.h" // _Py_INCREF_DICT() #include "pycore_function.h" // _PyFunction_Vectorcall #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" @@ -1240,6 +1242,58 @@ PyTypeObject PyFunction_Type = { }; +int +_PyFunction_VerifyStateless(PyThreadState *tstate, PyObject *func) +{ + assert(!PyErr_Occurred()); + assert(PyFunction_Check(func)); + + // Check the globals. + PyObject *globalsns = PyFunction_GET_GLOBALS(func); + if (globalsns != NULL && !PyDict_Check(globalsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported globals %R", globalsns); + return -1; + } + // Check the builtins. + PyObject *builtinsns = PyFunction_GET_BUILTINS(func); + if (builtinsns != NULL && !PyDict_Check(builtinsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported builtins %R", builtinsns); + return -1; + } + // Disallow __defaults__. + PyObject *defaults = PyFunction_GET_DEFAULTS(func); + if (defaults != NULL && defaults != Py_None && PyDict_Size(defaults) > 0) + { + _PyErr_SetString(tstate, PyExc_ValueError, "defaults not supported"); + return -1; + } + // Disallow __kwdefaults__. + PyObject *kwdefaults = PyFunction_GET_KW_DEFAULTS(func); + if (kwdefaults != NULL && kwdefaults != Py_None + && PyDict_Size(kwdefaults) > 0) + { + _PyErr_SetString(tstate, PyExc_ValueError, + "keyword defaults not supported"); + return -1; + } + // Disallow __closure__. + PyObject *closure = PyFunction_GET_CLOSURE(func); + if (closure != NULL && closure != Py_None && PyTuple_GET_SIZE(closure) > 0) + { + _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); + return -1; + } + // Check the code. + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + if (_PyCode_VerifyStateless(tstate, co, NULL, globalsns, builtinsns) < 0) { + return -1; + } + return 0; +} + + static int functools_copy_attr(PyObject *wrapper, PyObject *wrapped, PyObject *name) { From 9e1aa354aed03cff100b3ff0d1d6d13ecbb370ab Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 03:37:03 +0200 Subject: [PATCH 008/713] [3.14] gh-133551: Support t-strings in annotationlib (GH-133553) (#133628) gh-133551: Support t-strings in annotationlib (GH-133553) I don't know why you'd use t-strings in annotations, but now if you do, the STRING format will do a great job of recovering the source code. (cherry picked from commit 90f476e0f8dbb3a8603f67200c2422fb098c166c) Co-authored-by: Jelle Zijlstra --- Lib/annotationlib.py | 36 ++++++++++++++++++ Lib/test/.ruff.toml | 3 +- Lib/test/test_annotationlib.py | 38 +++++++++++++++++++ ...-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst | 2 + 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c0b1d4395d14ed..32b8553458930c 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -305,6 +305,9 @@ def __repr__(self): return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" +_Template = type(t"") + + class _Stringifier: # Must match the slots on ForwardRef, so we can turn an instance of one into an # instance of the other in place. @@ -341,6 +344,8 @@ def __convert_to_ast(self, other): if isinstance(other.__ast_node__, str): return ast.Name(id=other.__ast_node__), other.__extra_names__ return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None elif ( # In STRING format we don't bother with the create_unique_name() dance; # it's better to emit the repr() of the object instead of an opaque name. @@ -560,6 +565,32 @@ def unary_op(self): del _make_unary_op +def _template_to_ast(template): + values = [] + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + # Interpolation, but we don't want to import the string module + case _: + interp = ast.Interpolation( + str=part.expression, + value=ast.parse(part.expression), + conversion=( + ord(part.conversion) + if part.conversion is not None + else -1 + ), + format_spec=( + ast.Constant(value=part.format_spec) + if part.format_spec != "" + else None + ), + ) + values.append(interp) + return ast.TemplateStr(values=values) + + class _StringifierDict(dict): def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) @@ -784,6 +815,8 @@ def _stringify_single(anno): # We have to handle str specially to support PEP 563 stringified annotations. elif isinstance(anno, str): return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) else: return repr(anno) @@ -976,6 +1009,9 @@ def type_repr(value): if value.__module__ == "builtins": return value.__qualname__ return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) if value is ...: return "..." return repr(value) diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index a1eac32a83aae3..7aa8a4785d6844 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -9,8 +9,9 @@ extend-exclude = [ "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", # SyntaxError because of t-strings - "test_tstring.py", + "test_annotationlib.py", "test_string/test_templatelib.py", + "test_tstring.py", # New grammar constructions may not yet be recognized by Ruff, # and tests re-use the same names as only the grammar is being checked. "test_grammar.py", diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index c3c245ddaf86d1..4af97c82de9d46 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,6 +7,7 @@ import functools import itertools import pickle +from string.templatelib import Interpolation, Template import typing import unittest from annotationlib import ( @@ -273,6 +274,43 @@ def f( }, ) + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + def test_getitem(self): def f(x: undef1[str, undef2]): pass diff --git a/Misc/NEWS.d/next/Library/2025-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst b/Misc/NEWS.d/next/Library/2025-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst new file mode 100644 index 00000000000000..7fedc0818dc469 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst @@ -0,0 +1,2 @@ +Support t-strings (:pep:`750`) in :mod:`annotationlib`. Patch by Jelle +Zijlstra. From ca3e3b8a272c06ef31c36365b2ae39f457c747a0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 03:40:41 +0200 Subject: [PATCH 009/713] [3.14] gh-127833: lexical analysis: Add backticks to BOM example (GH-132407) (#133632) gh-127833: lexical analysis: Add backticks to BOM example (GH-132407) (cherry picked from commit 0552ce0fb24d33b51fdd02a589ede95aabc0d8b5) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/reference/lexical_analysis.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index ff801a7d4fc494..b8453946362867 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -99,7 +99,7 @@ which is recognized by Bram Moolenaar's VIM. If no encoding declaration is found, the default encoding is UTF-8. If the implicit or explicit encoding of a file is UTF-8, an initial UTF-8 byte-order -mark (b'\xef\xbb\xbf') is ignored rather than being a syntax error. +mark (``b'\xef\xbb\xbf'``) is ignored rather than being a syntax error. If an encoding is declared, the encoding name must be recognized by Python (see :ref:`standard-encodings`). The From aadfc2cbc7d98294b3a2ad0e72fe36f5527e21f7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 03:54:42 +0200 Subject: [PATCH 010/713] [3.14] gh-133555: Allow regenerating the parser with Python < 3.14 (GH-133557) (#133630) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-133555: Allow regenerating the parser with Python < 3.14 (GH-133557) (cherry picked from commit b48599b8017f896c965e89c402bb70feef49808a) Co-authored-by: Alex Prengère <2138730+alexprengere@users.noreply.github.com> Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Grammar/Tokens | 5 +++++ Tools/peg_generator/pegen/parser_generator.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/Grammar/Tokens b/Grammar/Tokens index e40a4437afb009..0547e6ed08f79a 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,3 +1,8 @@ +# When adding new tokens, remember to update the PEG generator in +# Tools/peg_generator/pegen/parser_generator.py +# This will ensure that older versions of Python can generate a Python parser +# using "python -m pegen python ". + ENDMARKER NAME NUMBER diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 6ce0649aefe7ff..52ae743c26b6b8 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -81,6 +81,11 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.tokens.add("FSTRING_START") self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") + # If python < 3.14 add the virtual tstring tokens + if sys.version_info < (3, 14, 0, 'beta', 1): + self.tokens.add("TSTRING_START") + self.tokens.add("TSTRING_END") + self.tokens.add("TSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: From f12e576128b2d8b201a9a59fa75296085212a4df Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 05:01:33 +0200 Subject: [PATCH 011/713] [3.14] gh-133551: Skip annotationlib for now in ast roundtrip tests (GH-133634) (#133636) gh-133551: Skip annotationlib for now in ast roundtrip tests (GH-133634) (cherry picked from commit 0a3ccb8dff2bd140f800d9e0b00549d8f1ead929) Co-authored-by: Jelle Zijlstra --- Lib/test/test_unparse.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d3af7a8489e650..5616129eb63c2f 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -942,6 +942,7 @@ def files_to_test(cls): for directory in cls.test_directories for item in directory.glob("*.py") if not item.name.startswith("bad") + and item.name != "annotationlib.py" # gh-133581: t"" does not roundtrip ] # Test limited subset of files unless the 'cpu' resource is specified. From e8c279f394d3b6f834e10a18ff0a7ee65630db0b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 08:32:05 +0200 Subject: [PATCH 012/713] [3.14] gh-133403: Type `Tools/build/update_file.py` and check it with `mypy` (GH-133404) (#133603) gh-133403: Type `Tools/build/update_file.py` and check it with `mypy` (GH-133404) (cherry picked from commit 50b52cba2d13a1854bc835412ac3f3c0ad42b5ba) Co-authored-by: sobolevn Co-authored-by: Zachary Ware --- Tools/build/mypy.ini | 6 +++++- Tools/build/update_file.py | 24 +++++++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 06224163884a98..db546c6fb3481c 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -1,7 +1,9 @@ [mypy] files = Tools/build/compute-changes.py, - Tools/build/generate_sbom.py + Tools/build/generate_sbom.py, + Tools/build/update_file.py + pretty = True # Make sure Python can still be built @@ -10,6 +12,8 @@ python_version = 3.10 # ...And be strict: strict = True +strict_bytes = True +local_partial_types = True extra_checks = True enable_error_code = ignore-without-code,redundant-expr,truthy-bool,possibly-undefined warn_unreachable = True diff --git a/Tools/build/update_file.py b/Tools/build/update_file.py index b4182c1d0cb638..b4a5fb6e778ae8 100644 --- a/Tools/build/update_file.py +++ b/Tools/build/update_file.py @@ -6,14 +6,27 @@ actually change the in-tree generated code. """ +from __future__ import annotations + import contextlib import os import os.path import sys +TYPE_CHECKING = False +if TYPE_CHECKING: + import typing + from collections.abc import Iterator + from io import TextIOWrapper + + _Outcome: typing.TypeAlias = typing.Literal['created', 'updated', 'same'] + @contextlib.contextmanager -def updating_file_with_tmpfile(filename, tmpfile=None): +def updating_file_with_tmpfile( + filename: str, + tmpfile: str | None = None, +) -> Iterator[tuple[TextIOWrapper, TextIOWrapper]]: """A context manager for updating a file via a temp file. The context manager provides two open files: the source file open @@ -46,13 +59,18 @@ def updating_file_with_tmpfile(filename, tmpfile=None): update_file_with_tmpfile(filename, tmpfile) -def update_file_with_tmpfile(filename, tmpfile, *, create=False): +def update_file_with_tmpfile( + filename: str, + tmpfile: str, + *, + create: bool = False, +) -> _Outcome: try: targetfile = open(filename, 'rb') except FileNotFoundError: if not create: raise # re-raise - outcome = 'created' + outcome: _Outcome = 'created' os.replace(tmpfile, filename) else: with targetfile: From c6a56e3d91be353c657b517f6b6394ec5323e5e7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 09:11:49 +0200 Subject: [PATCH 013/713] [3.14] gh-132983: Remove pyzstd in identifiers (GH-133535) (#133629) --- Modules/_zstd/_zstdmodule.h | 13 ------------- Modules/_zstd/compressor.c | 16 ++++++++-------- Modules/_zstd/decompressor.c | 12 ++++++------ 3 files changed, 14 insertions(+), 27 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 9322ee259c5124..1e9e4c75056831 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -172,19 +172,6 @@ set_parameter_error(const _zstd_state* const state, int is_compress, static const char init_twice_msg[] = "__init__ method is called twice."; -extern int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict); - -extern int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict); - -extern int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char *arg_type); - -extern int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options); - extern PyObject * decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, Py_ssize_t max_length, diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b735981e7476d5..9a3d8dedcf235e 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -25,9 +25,9 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_ty #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) -int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char* arg_type) +static int +_zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, + const char *arg_name, const char* arg_type) { size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -197,8 +197,8 @@ _get_CDict(ZstdDict *self, int compressionLevel) return cdict; } -int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { +static int +_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -385,20 +385,20 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, /* Set compressLevel/options to compression context */ if (level != Py_None) { - if (_PyZstd_set_c_parameters(self, level, "level", "int") < 0) { + if (_zstd_set_c_parameters(self, level, "level", "int") < 0) { return -1; } } if (options != Py_None) { - if (_PyZstd_set_c_parameters(self, options, "options", "dict") < 0) { + if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) { return -1; } } /* Load dictionary to compression context */ if (zstd_dict != Py_None) { - if (_PyZstd_load_c_dict(self, zstd_dict) < 0) { + if (_zstd_load_c_dict(self, zstd_dict) < 0) { return -1; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index a4be180c0088fc..748947af79c3f7 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -61,8 +61,8 @@ _get_DDict(ZstdDict *self) } /* Set decompression parameters to decompression context */ -int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) +static int +_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) { size_t zstd_ret; PyObject *key, *value; @@ -120,8 +120,8 @@ _PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) } /* Load dictionary or prefix to decompression context */ -int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +static int +_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -709,7 +709,7 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /* Load dictionary to decompression context */ if (zstd_dict != Py_None) { - if (_PyZstd_load_d_dict(self, zstd_dict) < 0) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { return -1; } @@ -720,7 +720,7 @@ _zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, /* Set option to decompression context */ if (options != Py_None) { - if (_PyZstd_set_d_parameters(self, options) < 0) { + if (_zstd_set_d_parameters(self, options) < 0) { return -1; } } From effe8d4971f6441efe897b2ba4a991ee39e73f1d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 10:26:14 +0200 Subject: [PATCH 014/713] [3.14] gh-125028: Prohibit placeholders in partial keywords (GH-126062) (GH-133645) (cherry picked from commit afed5f88359c73f798ff6f0064e37ac1a1f0759b) Co-authored-by: dgpb <3577712+dg-pb@users.noreply.github.com> --- Doc/library/functools.rst | 3 +-- Lib/functools.py | 3 +++ Lib/test/test_functools.py | 19 +++++++++++++++++++ ...-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst | 1 + Modules/_functoolsmodule.c | 13 +++++++++++++ 5 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2024-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3a933dff057bbb..3e75621be6dad3 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -403,8 +403,7 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - :data:`!Placeholder` has no special treatment when used in a keyword - argument to :func:`!partial`. + :data:`!Placeholder` cannot be passed to :func:`!partial` as a keyword argument. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. diff --git a/Lib/functools.py b/Lib/functools.py index 714070c6ac9460..7f0eac3f650209 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -323,6 +323,9 @@ def _partial_new(cls, func, /, *args, **keywords): "or a descriptor") if args and args[-1] is Placeholder: raise TypeError("trailing Placeholders are not allowed") + for value in keywords.values(): + if value is Placeholder: + raise TypeError("Placeholder cannot be passed as a keyword argument") if isinstance(func, base_cls): pto_phcount = func._phcount tot_args = func.args diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 2e794b0fc95a22..f7e09fd771eaf2 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -21,6 +21,7 @@ import contextlib from inspect import Signature +from test.support import ALWAYS_EQ from test.support import import_helper from test.support import threading_helper from test.support import cpython_only @@ -244,6 +245,13 @@ def test_placeholders(self): actual_args, actual_kwds = p('x', 'y') self.assertEqual(actual_args, ('x', 0, 'y', 1)) self.assertEqual(actual_kwds, {}) + # Checks via `is` and not `eq` + # thus ALWAYS_EQ isn't treated as Placeholder + p = self.partial(capture, ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(len(actual_args), 1) + self.assertIs(actual_args[0], ALWAYS_EQ) + self.assertEqual(actual_kwds, {}) def test_placeholders_optimization(self): PH = self.module.Placeholder @@ -260,6 +268,17 @@ def test_placeholders_optimization(self): self.assertEqual(p2.args, (PH, 0)) self.assertEqual(p2(1), ((1, 0), {})) + def test_placeholders_kw_restriction(self): + PH = self.module.Placeholder + with self.assertRaisesRegex(TypeError, "Placeholder"): + self.partial(capture, a=PH) + # Passes, as checks via `is` and not `eq` + p = self.partial(capture, a=ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(actual_args, ()) + self.assertEqual(len(actual_kwds), 1) + self.assertIs(actual_kwds['a'], ALWAYS_EQ) + def test_construct_placeholder_singleton(self): PH = self.module.Placeholder tp = type(PH) diff --git a/Misc/NEWS.d/next/Library/2024-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst b/Misc/NEWS.d/next/Library/2024-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst new file mode 100644 index 00000000000000..09ebee4d41b68e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst @@ -0,0 +1 @@ +:data:`functools.Placeholder` cannot be passed to :func:`functools.partial` as a keyword argument. diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e6c454faf4b16f..899eef50ecc600 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -196,6 +196,19 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + /* keyword Placeholder prohibition */ + if (kw != NULL) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(kw, &pos, &key, &val)) { + if (val == phold) { + PyErr_SetString(PyExc_TypeError, + "Placeholder cannot be passed as a keyword argument"); + return NULL; + } + } + } + /* check wrapped function / object */ pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); From d35c0e48da81f76db54e9165f01c5b3c5f8c92e8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 10:34:20 +0200 Subject: [PATCH 015/713] [3.14] gh-133641: Doc: Add missing source link in ``concurrent.futures`` (GH-133642) (#133643) gh-133641: Doc: Add missing source link in ``concurrent.futures`` (GH-133642) (cherry picked from commit 4fcd37756382206e5a4a387cec9266815adca007) Co-authored-by: Jonas Obrist Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/concurrent.futures.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 7efae9e628b828..3c8d9ab111e09e 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -6,8 +6,9 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` +**Source code:** :source:`Lib/concurrent/futures/thread.py`, +:source:`Lib/concurrent/futures/process.py`, +and :source:`Lib/concurrent/futures/interpreter.py` -------------- From 2de8a8b7e93e9bcddfea56b4f58a5e3efb5c284d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 10:50:57 +0200 Subject: [PATCH 016/713] [3.14] gh-133639: Fix `test_auto_indent_default()` doesn't run `input_code` (GH-133640) (#133646) --- Lib/test/test_pyrepl/test_pyrepl.py | 5 +++++ .../Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst | 2 ++ 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 93029ab6e080ba..fc8114891d12dd 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -452,6 +452,11 @@ def test_auto_indent_default(self): ) # fmt: on + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + def test_auto_indent_continuation(self): # auto indenting according to previous user indentation # fmt: off diff --git a/Misc/NEWS.d/next/Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst b/Misc/NEWS.d/next/Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst new file mode 100644 index 00000000000000..68826cd95fa2b0 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst @@ -0,0 +1,2 @@ +Fix ``TestPyReplAutoindent.test_auto_indent_default()`` doesn't run +``input_code``. From 3cd4c2db7bebc63b9773d0cda5ae1ea7b3a1b634 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 11:28:58 +0200 Subject: [PATCH 017/713] [3.14] Doc: Allow translating a code block in the tutorial (GH-131353) (#133648) Doc: Allow translating a code block in the tutorial (GH-131353) (cherry picked from commit 3224b99872d8ed030181e5d009ea64e8672a68ee) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Chris Markiewicz Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/tutorial/introduction.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index bec5da8fd759ac..cdb35da7bc95ba 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -147,6 +147,8 @@ Python can manipulate text (represented by type :class:`str`, so-called "``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double quotes (``"..."``) with the same result [#]_. +.. code-block:: pycon + >>> 'spam eggs' # single quotes 'spam eggs' >>> "Paris rabbit got your back :)! Yay!" # double quotes From a0203362568ade0bc30f22a8abb22a3387c329f1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 11:46:47 +0200 Subject: [PATCH 018/713] [3.14] gh-127833: Add links to token types to the lexical analysis intro (GH-131468) (#133652) gh-127833: Add links to token types to the lexical analysis intro (GH-131468) (cherry picked from commit 45bb5ba61a8d6dcaa44312b67fe6bade5e2ad992) Co-authored-by: Petr Viktorin Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/reference/lexical_analysis.rst | 62 +++++++++++++++++++----------- 1 file changed, 39 insertions(+), 23 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index b8453946362867..001e2547fe8031 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -35,11 +35,11 @@ Logical lines .. index:: logical line, physical line, line joining, NEWLINE token -The end of a logical line is represented by the token NEWLINE. Statements -cannot cross logical line boundaries except where NEWLINE is allowed by the -syntax (e.g., between statements in compound statements). A logical line is -constructed from one or more *physical lines* by following the explicit or -implicit *line joining* rules. +The end of a logical line is represented by the token :data:`~token.NEWLINE`. +Statements cannot cross logical line boundaries except where :data:`!NEWLINE` +is allowed by the syntax (e.g., between statements in compound statements). +A logical line is constructed from one or more *physical lines* by following +the explicit or implicit *line joining* rules. .. _physical-lines: @@ -160,11 +160,12 @@ Blank lines .. index:: single: blank line A logical line that contains only spaces, tabs, formfeeds and possibly a -comment, is ignored (i.e., no NEWLINE token is generated). During interactive -input of statements, handling of a blank line may differ depending on the -implementation of the read-eval-print loop. In the standard interactive -interpreter, an entirely blank logical line (i.e. one containing not even -whitespace or a comment) terminates a multi-line statement. +comment, is ignored (i.e., no :data:`~token.NEWLINE` token is generated). +During interactive input of statements, handling of a blank line may differ +depending on the implementation of the read-eval-print loop. +In the standard interactive interpreter, an entirely blank logical line (that +is, one containing not even whitespace or a comment) terminates a multi-line +statement. .. _indentation: @@ -202,19 +203,20 @@ the space count to zero). .. index:: INDENT token, DEDENT token -The indentation levels of consecutive lines are used to generate INDENT and -DEDENT tokens, using a stack, as follows. +The indentation levels of consecutive lines are used to generate +:data:`~token.INDENT` and :data:`~token.DEDENT` tokens, using a stack, +as follows. Before the first line of the file is read, a single zero is pushed on the stack; this will never be popped off again. The numbers pushed on the stack will always be strictly increasing from bottom to top. At the beginning of each logical line, the line's indentation level is compared to the top of the stack. If it is equal, nothing happens. If it is larger, it is pushed on the stack, and -one INDENT token is generated. If it is smaller, it *must* be one of the +one :data:`!INDENT` token is generated. If it is smaller, it *must* be one of the numbers occurring on the stack; all numbers on the stack that are larger are -popped off, and for each number popped off a DEDENT token is generated. At the -end of the file, a DEDENT token is generated for each number remaining on the -stack that is larger than zero. +popped off, and for each number popped off a :data:`!DEDENT` token is generated. +At the end of the file, a :data:`!DEDENT` token is generated for each number +remaining on the stack that is larger than zero. Here is an example of a correctly (though confusingly) indented piece of Python code:: @@ -254,8 +256,18 @@ Whitespace between tokens Except at the beginning of a logical line or in string literals, the whitespace characters space, tab and formfeed can be used interchangeably to separate tokens. Whitespace is needed between two tokens only if their concatenation -could otherwise be interpreted as a different token (e.g., ab is one token, but -a b is two tokens). +could otherwise be interpreted as a different token. For example, ``ab`` is one +token, but ``a b`` is two tokens. However, ``+a`` and ``+ a`` both produce +two tokens, ``+`` and ``a``, as ``+a`` is not a valid token. + + +.. _endmarker-token: + +End marker +---------- + +At the end of non-interactive input, the lexical analyzer generates an +:data:`~token.ENDMARKER` token. .. _other-tokens: @@ -263,11 +275,15 @@ a b is two tokens). Other tokens ============ -Besides NEWLINE, INDENT and DEDENT, the following categories of tokens exist: -*identifiers*, *keywords*, *literals*, *operators*, and *delimiters*. Whitespace -characters (other than line terminators, discussed earlier) are not tokens, but -serve to delimit tokens. Where ambiguity exists, a token comprises the longest -possible string that forms a legal token, when read from left to right. +Besides :data:`~token.NEWLINE`, :data:`~token.INDENT` and :data:`~token.DEDENT`, +the following categories of tokens exist: +*identifiers* and *keywords* (:data:`~token.NAME`), *literals* (such as +:data:`~token.NUMBER` and :data:`~token.STRING`), and other symbols +(*operators* and *delimiters*, :data:`~token.OP`). +Whitespace characters (other than logical line terminators, discussed earlier) +are not tokens, but serve to delimit tokens. +Where ambiguity exists, a token comprises the longest possible string that +forms a legal token, when read from left to right. .. _identifiers: From 0f7046b18760c42da8676fea0421b25d67fb32e6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 11:54:44 +0200 Subject: [PATCH 019/713] [3.14] gh-133583: Add support for fixed size unsigned integers in argument parsing (GH-133584) (GH-133650) * Add Argument Clinic converters: uint8, uint16, uint32, uint64. * Add private C API: _PyLong_UInt8_Converter(), _PyLong_UInt16_Converter(), _PyLong_UInt32_Converter(), _PyLong_UInt64_Converter(). (cherry picked from commit 4c914e7a36033d5e8d52cdef3a4063eb9b012ed2) Co-authored-by: Serhiy Storchaka --- Include/internal/pycore_long.h | 5 ++ Lib/test/test_clinic.py | 4 + Modules/_lzmamodule.c | 25 +++--- Modules/clinic/socketmodule.c.h | 3 +- Modules/socketmodule.c | 15 +--- Objects/longobject.c | 4 + Tools/clinic/libclinic/converters.py | 124 +++++++++++++-------------- 7 files changed, 87 insertions(+), 93 deletions(-) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ed6c435316708e..3196d1b82084b9 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -158,6 +158,11 @@ PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); // Export for '_testclinic' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt8_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt16_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt32_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt64_Converter(PyObject *, void *); + /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. * 2: Set to 1 for the small ints diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 0c99620e27cde4..90b7d343cea583 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2835,6 +2835,10 @@ def test_cli_converters(self): "size_t", "slice_index", "str", + "uint16", + "uint32", + "uint64", + "uint8", "unicode", "unsigned_char", "unsigned_int", diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f9b4c2a170e9c5..462c2181fa6036 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -17,6 +17,7 @@ #include +#include "pycore_long.h" // _PyLong_UInt32_Converter() // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -223,8 +224,6 @@ FUNCNAME(PyObject *obj, void *ptr) \ return 1; \ } -INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter) -INT_TYPE_CONVERTER_FUNC(uint64_t, uint64_converter) INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter) INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter) INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) @@ -254,7 +253,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) return NULL; } if (preset_obj != NULL) { - int ok = uint32_converter(preset_obj, &preset); + int ok = _PyLong_UInt32_Converter(preset_obj, &preset); Py_DECREF(preset_obj); if (!ok) { return NULL; @@ -275,14 +274,14 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, - uint32_converter, &options->dict_size, - uint32_converter, &options->lc, - uint32_converter, &options->lp, - uint32_converter, &options->pb, + _PyLong_UInt32_Converter, &options->dict_size, + _PyLong_UInt32_Converter, &options->lc, + _PyLong_UInt32_Converter, &options->lp, + _PyLong_UInt32_Converter, &options->pb, lzma_mode_converter, &options->mode, - uint32_converter, &options->nice_len, + _PyLong_UInt32_Converter, &options->nice_len, lzma_mf_converter, &options->mf, - uint32_converter, &options->depth)) { + _PyLong_UInt32_Converter, &options->depth)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); @@ -301,7 +300,7 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) lzma_options_delta *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &dist)) { + &id, _PyLong_UInt32_Converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); return NULL; @@ -325,7 +324,7 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) lzma_options_bcj *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &start_offset)) { + &id, _PyLong_UInt32_Converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); return NULL; @@ -806,7 +805,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } - if (preset_obj != Py_None && !uint32_converter(preset_obj, &preset)) { + if (preset_obj != Py_None && !_PyLong_UInt32_Converter(preset_obj, &preset)) { return NULL; } @@ -1226,7 +1225,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, "Cannot specify memory limit with FORMAT_RAW"); return NULL; } - if (!uint64_converter(memlimit, &memlimit_)) { + if (!_PyLong_UInt64_Converter(memlimit, &memlimit_)) { return NULL; } } diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 70ebbaa876b32b..573903be87e6ea 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UInt16_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_socket_socket_close__doc__, @@ -369,4 +370,4 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=c971b79d2193b426 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=07776dd21d1e3b56 input=a9049054013a1b77]*/ diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 47958379263793..77bc4314808235 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -638,33 +638,22 @@ _PyLong_##NAME##_Converter(PyObject *obj, void *ptr) \ return 1; \ } -UNSIGNED_INT_CONVERTER(UInt16, uint16_t) -UNSIGNED_INT_CONVERTER(UInt32, uint32_t) - #if defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) # ifdef MS_WINDOWS UNSIGNED_INT_CONVERTER(NetIfindex, NET_IFINDEX) # else - UNSIGNED_INT_CONVERTER(NetIfindex, unsigned int) +# define _PyLong_NetIfindex_Converter _PyLong_UnsignedInt_Converter # define NET_IFINDEX unsigned int # endif #endif // defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) /*[python input] -class uint16_converter(CConverter): - type = "uint16_t" - converter = '_PyLong_UInt16_Converter' - -class uint32_converter(CConverter): - type = "uint32_t" - converter = '_PyLong_UInt32_Converter' - class NET_IFINDEX_converter(CConverter): type = "NET_IFINDEX" converter = '_PyLong_NetIfindex_Converter' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3de2e4a03fbf83b8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=1cf809c40a407c34]*/ /*[clinic input] module _socket diff --git a/Objects/longobject.c b/Objects/longobject.c index 40d90ecf4fa068..0b2dfa003fac53 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -1760,6 +1760,10 @@ UNSIGNED_INT_CONVERTER(UnsignedInt, unsigned int) UNSIGNED_INT_CONVERTER(UnsignedLong, unsigned long) UNSIGNED_INT_CONVERTER(UnsignedLongLong, unsigned long long) UNSIGNED_INT_CONVERTER(Size_t, size_t) +UNSIGNED_INT_CONVERTER(UInt8, uint8_t) +UNSIGNED_INT_CONVERTER(UInt16, uint16_t) +UNSIGNED_INT_CONVERTER(UInt32, uint32_t) +UNSIGNED_INT_CONVERTER(UInt64, uint64_t) #define CHECK_BINOP(v,w) \ diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 633fb5f56a6693..39d0ac557a60f5 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -17,6 +17,54 @@ TypeSet = set[bltns.type[object]] +class BaseUnsignedIntConverter(CConverter): + + def use_converter(self) -> None: + if self.converter: + self.add_include('pycore_long.h', + f'{self.converter}()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + {{{{ + Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_REJECT_NEGATIVE | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) {{{{ + goto exit; + }}}} + if ((size_t)_bytes > sizeof({type})) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C {type}"); + goto exit; + }}}} + }}}} + """, + argname=argname, + type=self.type) + + +class uint8_converter(BaseUnsignedIntConverter): + type = "uint8_t" + converter = '_PyLong_UInt8_Converter' + +class uint16_converter(BaseUnsignedIntConverter): + type = "uint16_t" + converter = '_PyLong_UInt16_Converter' + +class uint32_converter(BaseUnsignedIntConverter): + type = "uint32_t" + converter = '_PyLong_UInt32_Converter' + +class uint64_converter(BaseUnsignedIntConverter): + type = "uint64_t" + converter = '_PyLong_UInt64_Converter' + + class bool_converter(CConverter): type = 'int' default_type = bool @@ -211,29 +259,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -def format_inline_unsigned_int_converter(self: CConverter, argname: str) -> str: - return self.format_code(""" - {{{{ - Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), - Py_ASNATIVEBYTES_NATIVE_ENDIAN | - Py_ASNATIVEBYTES_ALLOW_INDEX | - Py_ASNATIVEBYTES_REJECT_NEGATIVE | - Py_ASNATIVEBYTES_UNSIGNED_BUFFER); - if (_bytes < 0) {{{{ - goto exit; - }}}} - if ((size_t)_bytes > sizeof({type})) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C {type}"); - goto exit; - }}}} - }}}} - """, - argname=argname, - type=self.type) - - -class unsigned_short_converter(CConverter): +class unsigned_short_converter(BaseUnsignedIntConverter): type = 'unsigned short' default_type = int c_ignored_default = "0" @@ -244,11 +270,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedShort_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedShort_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'H': return self.format_code(""" @@ -258,9 +279,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) @add_legacy_c_converter('C', accept={str}) @@ -311,7 +330,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_int_converter(CConverter): +class unsigned_int_converter(BaseUnsignedIntConverter): type = 'unsigned int' default_type = int c_ignored_default = "0" @@ -322,11 +341,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedInt_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedInt_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'I': return self.format_code(""" @@ -336,9 +350,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class long_converter(CConverter): @@ -359,7 +371,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_converter(CConverter): +class unsigned_long_converter(BaseUnsignedIntConverter): type = 'unsigned long' default_type = int c_ignored_default = "0" @@ -370,11 +382,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'k': return self.format_code(""" @@ -387,9 +394,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname, bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class long_long_converter(CConverter): @@ -410,7 +415,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_long_converter(CConverter): +class unsigned_long_long_converter(BaseUnsignedIntConverter): type = 'unsigned long long' default_type = int c_ignored_default = "0" @@ -421,11 +426,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedLongLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLongLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'K': return self.format_code(""" @@ -438,9 +438,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname, bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class Py_ssize_t_converter(CConverter): @@ -557,15 +555,11 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname) -class size_t_converter(CConverter): +class size_t_converter(BaseUnsignedIntConverter): type = 'size_t' converter = '_PyLong_Size_t_Converter' c_ignored_default = "0" - def use_converter(self) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': return self.format_code(""" @@ -575,9 +569,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class fildes_converter(CConverter): From caeb422cbca5a4470d406b52f3a712305214982e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 13:36:38 +0200 Subject: [PATCH 020/713] gh-133597: Fix memory leak if error occurred in _sys_getwindowsversion_from_kernel32 (GH-133598) (cherry picked from commit 0ec8fc83a83914d2ae97e52529e98ce63aaf831c) Co-authored-by: Sergey Miryanov --- Python/sysmodule.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 00dce4527fbb90..41b9a6b276a3b1 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1670,6 +1670,9 @@ _sys_getwindowsversion_from_kernel32(void) !GetFileVersionInfoW(kernel32_path, 0, verblock_size, verblock) || !VerQueryValueW(verblock, L"", (LPVOID)&ffi, &ffi_len)) { PyErr_SetFromWindowsErr(0); + if (verblock) { + PyMem_RawFree(verblock); + } return NULL; } From 863722be6b7c31c69dbf931581a531979dd28203 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 15:24:38 +0200 Subject: [PATCH 021/713] [3.14] gh-133454: Mark tests with many threads that use much memory as bigmem (GH-133456) (GH-133663) (cherry picked from commit 26839eae2079bd797491dfdcc00eaca1dbcbf77c) Co-authored-by: Serhiy Storchaka --- Lib/test/test_asyncio/test_ssl.py | 10 ++++++---- Lib/test/test_importlib/test_threaded_import.py | 15 ++++++++++----- Lib/test/test_threadedtempfile.py | 4 +++- Lib/test/test_threading.py | 6 ++++-- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 986ecc2c5a964b..3a7185cd8974d0 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -195,9 +195,10 @@ async def wait_closed(self, obj): except (BrokenPipeError, ConnectionError): pass - def test_create_server_ssl_1(self): + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_1(self, size): CNT = 0 # number of clients that were successful - TOTAL_CNT = 25 # total number of clients that test will create + TOTAL_CNT = size # total number of clients that test will create TIMEOUT = support.LONG_TIMEOUT # timeout for this test A_DATA = b'A' * 1024 * BUF_MULTIPLIER @@ -1038,9 +1039,10 @@ async def run_main(): self.loop.run_until_complete(run_main()) - def test_create_server_ssl_over_ssl(self): + @support.bigmemtest(size=25, memuse=90*2**20, dry_run=False) + def test_create_server_ssl_over_ssl(self, size): CNT = 0 # number of clients that were successful - TOTAL_CNT = 25 # total number of clients that test will create + TOTAL_CNT = size # total number of clients that test will create TIMEOUT = support.LONG_TIMEOUT # timeout for this test A_DATA = b'A' * 1024 * BUF_MULTIPLIER diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 9af1e4d505c66e..f78dc399720c86 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -135,10 +135,12 @@ def check_parallel_module_init(self, mock_os): if verbose: print("OK.") - def test_parallel_module_init(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_module_init(self, size): self.check_parallel_module_init() - def test_parallel_meta_path(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_meta_path(self, size): finder = Finder() sys.meta_path.insert(0, finder) try: @@ -148,7 +150,8 @@ def test_parallel_meta_path(self): finally: sys.meta_path.remove(finder) - def test_parallel_path_hooks(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_path_hooks(self, size): # Here the Finder instance is only used to check concurrent calls # to path_hook(). finder = Finder() @@ -242,13 +245,15 @@ def target(): __import__(TESTFN) del sys.modules[TESTFN] - def test_concurrent_futures_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_concurrent_futures_circular_import(self, size): # Regression test for bpo-43515 fn = os.path.join(os.path.dirname(__file__), 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - def test_multiprocessing_pool_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_multiprocessing_pool_circular_import(self, size): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), 'partial', 'pool_in_threads.py') diff --git a/Lib/test/test_threadedtempfile.py b/Lib/test/test_threadedtempfile.py index 420fc6ec8be3d8..acb427b0c78ae9 100644 --- a/Lib/test/test_threadedtempfile.py +++ b/Lib/test/test_threadedtempfile.py @@ -15,6 +15,7 @@ import tempfile +from test import support from test.support import threading_helper import unittest import io @@ -49,7 +50,8 @@ def run(self): class ThreadedTempFileTest(unittest.TestCase): - def test_main(self): + @support.bigmemtest(size=NUM_THREADS, memuse=60*2**20, dry_run=False) + def test_main(self, size): threads = [TempFileGreedy() for i in range(NUM_THREADS)] with threading_helper.start_threads(threads, startEvent.set): pass diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4ab38c2598b50a..abe63c10c0ac7c 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -530,7 +530,8 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) - def test_join_from_multiple_threads(self): + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) + def test_join_from_multiple_threads(self, size): # Thread.join() should be thread-safe errors = [] @@ -1431,7 +1432,8 @@ def worker(): self._run_and_join(script) @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_daemon_threads(self): + @support.bigmemtest(size=40, memuse=70*2**20, dry_run=False) + def test_4_daemon_threads(self, size): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. From 153496921a05e205d4c45c4a64888ccd43148897 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 15:40:44 +0200 Subject: [PATCH 022/713] [3.14] gh-133581: Improve AST unparsing of t-strings (GH-133635) (#133666) gh-133581: Improve AST unparsing of t-strings (GH-133635) (cherry picked from commit bfac7d2edc6e461e21b4762f8db03fca85f3a0be) Co-authored-by: Jelle Zijlstra --- Lib/_ast_unparse.py | 12 ++++-- Lib/test/test_future_stmt/test_future.py | 5 +++ Lib/test/test_unparse.py | 10 ++++- ...-05-07-19-16-41.gh-issue-133581.kERUCJ.rst | 4 ++ Python/ast_unparse.c | 38 ++++++++++++------- 5 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-07-19-16-41.gh-issue-133581.kERUCJ.rst diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 0b669edb2ffec6..c25066eb107de1 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -627,6 +627,9 @@ def _write_ftstring(self, values, prefix): self._ftstring_helper(fstring_parts) def _tstring_helper(self, node): + if not node.values: + self._write_ftstring([], "t") + return last_idx = 0 for i, value in enumerate(node.values): # This can happen if we have an implicit concat of a t-string @@ -679,9 +682,12 @@ def _unparse_interpolation_value(self, inner): unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) - def _write_interpolation(self, node): + def _write_interpolation(self, node, is_interpolation=False): with self.delimit("{", "}"): - expr = self._unparse_interpolation_value(node.value) + if is_interpolation: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") @@ -696,7 +702,7 @@ def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): - self._write_interpolation(node) + self._write_interpolation(node, is_interpolation=True) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 42c6cb3fefac33..71f1e616116d81 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -422,6 +422,11 @@ def test_annotations(self): eq('(((a)))', 'a') eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + eq("t''") + eq("t'{a + b}'") + eq("t'{a!s}'") + eq("t'{a:b}'") + eq("t'{a:b=}'") def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index 5616129eb63c2f..d4db5e60af7978 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -817,6 +817,15 @@ def test_type_params(self): self.check_ast_roundtrip("def f[T: int = int, **P = int, *Ts = *int]():\n pass") self.check_ast_roundtrip("class C[T: int = int, **P = int, *Ts = *int]():\n pass") + def test_tstr(self): + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + class ManualASTCreationTestCase(unittest.TestCase): """Test that AST nodes created without a type_params field unparse correctly.""" @@ -942,7 +951,6 @@ def files_to_test(cls): for directory in cls.test_directories for item in directory.glob("*.py") if not item.name.startswith("bad") - and item.name != "annotationlib.py" # gh-133581: t"" does not roundtrip ] # Test limited subset of files unless the 'cpu' resource is specified. diff --git a/Misc/NEWS.d/next/Library/2025-05-07-19-16-41.gh-issue-133581.kERUCJ.rst b/Misc/NEWS.d/next/Library/2025-05-07-19-16-41.gh-issue-133581.kERUCJ.rst new file mode 100644 index 00000000000000..3749904cd9b8f1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-07-19-16-41.gh-issue-133581.kERUCJ.rst @@ -0,0 +1,4 @@ +Improve unparsing of t-strings in :func:`ast.unparse` and ``from __future__ +import annotations``. Empty t-strings now round-trip correctly and +formatting in interpolations is preserved. +Patch by Jelle Zijlstra. diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index c121ec096aebf4..ae623e0b4171f8 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -702,6 +702,13 @@ append_templatestr(PyUnicodeWriter *writer, expr_ty e) Py_ssize_t last_idx = 0; Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values); + if (len == 0) { + int result = _write_values_subarray(writer, e->v.TemplateStr.values, + 0, len - 1, 't', arena); + _PyArena_Free(arena); + return result; + } + for (Py_ssize_t i = 0; i < len; i++) { expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i); @@ -774,32 +781,37 @@ append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) } static int -append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +append_interpolation_str(PyUnicodeWriter *writer, PyObject *str) { const char *outer_brace = "{"; - /* Grammar allows PR_TUPLE, but use >PR_TEST for adding parenthesis - around a lambda with ':' */ - PyObject *temp_fv_str = expr_as_unicode(e, PR_TEST + 1); - if (!temp_fv_str) { - return -1; - } - if (PyUnicode_Find(temp_fv_str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { + if (PyUnicode_Find(str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { /* Expression starts with a brace, split it with a space from the outer one. */ outer_brace = "{ "; } if (-1 == append_charp(writer, outer_brace)) { - Py_DECREF(temp_fv_str); return -1; } - if (-1 == PyUnicodeWriter_WriteStr(writer, temp_fv_str)) { - Py_DECREF(temp_fv_str); + if (-1 == PyUnicodeWriter_WriteStr(writer, str)) { return -1; } - Py_DECREF(temp_fv_str); return 0; } +static int +append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +{ + /* Grammar allows PR_TUPLE, but use >PR_TEST for adding parenthesis + around a lambda with ':' */ + PyObject *temp_fv_str = expr_as_unicode(e, PR_TEST + 1); + if (!temp_fv_str) { + return -1; + } + int result = append_interpolation_str(writer, temp_fv_str); + Py_DECREF(temp_fv_str); + return result; +} + static int append_interpolation_conversion(PyUnicodeWriter *writer, int conversion) { @@ -843,7 +855,7 @@ append_interpolation_format_spec(PyUnicodeWriter *writer, expr_ty e) static int append_interpolation(PyUnicodeWriter *writer, expr_ty e) { - if (-1 == append_interpolation_value(writer, e->v.Interpolation.value)) { + if (-1 == append_interpolation_str(writer, e->v.Interpolation.str)) { return -1; } From dc1a4dda88637d551d0be26632186be10343b098 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 17:19:58 +0200 Subject: [PATCH 023/713] [3.14] gh-131031: Fix test_pickle when invoked directly (GH-133356) (GH-133673) (cherry picked from commit e15bbfafbcc9153fa0b05beb806984e288fdefe4) Co-authored-by: Serhiy Storchaka --- Lib/test/pickletester.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index bdc7ef62943a28..dcba6369541049 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -2272,7 +2272,11 @@ def test_nonencodable_module_name_error(self): def test_nested_lookup_error(self): # Nested name does not exist - obj = REX('AbstractPickleTests.spam') + global TestGlobal + class TestGlobal: + class A: + pass + obj = REX('TestGlobal.A.B.C') obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): @@ -2280,9 +2284,9 @@ def test_nested_lookup_error(self): self.dumps(obj, proto) self.assertEqual(str(cm.exception), f"Can't pickle {obj!r}: " - f"it's not found as {__name__}.AbstractPickleTests.spam") + f"it's not found as {__name__}.TestGlobal.A.B.C") self.assertEqual(str(cm.exception.__context__), - "type object 'AbstractPickleTests' has no attribute 'spam'") + "type object 'A' has no attribute 'B'") obj.__module__ = None for proto in protocols: @@ -2290,21 +2294,25 @@ def test_nested_lookup_error(self): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests.spam") + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal.A.B.C") self.assertEqual(str(cm.exception.__context__), - "module '__main__' has no attribute 'AbstractPickleTests'") + "module '__main__' has no attribute 'TestGlobal'") def test_wrong_object_lookup_error(self): # Name is bound to different object - obj = REX('AbstractPickleTests') + global TestGlobal + class TestGlobal: + pass + obj = REX('TestGlobal') obj.__module__ = __name__ - AbstractPickleTests.ham = [] for proto in protocols: with self.subTest(proto=proto): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not the same object as {__name__}.AbstractPickleTests") + f"Can't pickle {obj!r}: " + f"it's not the same object as {__name__}.TestGlobal") self.assertIsNone(cm.exception.__context__) obj.__module__ = None @@ -2313,9 +2321,10 @@ def test_wrong_object_lookup_error(self): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests") + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal") self.assertEqual(str(cm.exception.__context__), - "module '__main__' has no attribute 'AbstractPickleTests'") + "module '__main__' has no attribute 'TestGlobal'") def test_local_lookup_error(self): # Test that whichmodule() errors out cleanly when looking up From 10595486860aa054c9d7ad236cc50064e7b800d5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 8 May 2025 18:05:34 +0200 Subject: [PATCH 024/713] [3.14] gh-132775: Add _PyCode_GetScriptXIData() (gh-133676) This converts functions, code, str, bytes, bytearray, and memoryview objects to PyCodeObject, and ensure that the object looks like a script. That means no args, no return, and no closure. _PyCode_GetPureScriptXIData() takes it a step further and ensures there are no globals. We also add _PyObject_SupportedAsScript() to the internal C-API. (cherry picked from commit c81fa2b9cd1ed4d4ff5c13984d6582c27e5f2633, AKA gh-133480) Co-authored-by: Eric Snow --- Include/internal/pycore_crossinterp.h | 8 ++ Include/internal/pycore_pythonrun.h | 1 + Lib/test/_code_definitions.py | 53 +++++++++++ Lib/test/test_code.py | 27 ++++++ Lib/test/test_crossinterp.py | 120 ++++++++++++++++++++++++ Modules/_testinternalcapi.c | 10 ++ Python/crossinterp.c | 127 ++++++++++++++++++++++++++ Python/pythonrun.c | 20 ++++ 8 files changed, 366 insertions(+) diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 9de61ef54125d5..9c9b2c2f9c599d 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -191,6 +191,14 @@ PyAPI_FUNC(int) _PyCode_GetXIData( PyThreadState *, PyObject *, _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetPureScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); /* using cross-interpreter data */ diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 0bfc5704dc4c59..7daed1326af8d5 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -25,6 +25,7 @@ extern int _PyRun_InteractiveLoopObject( PyObject *filename, PyCompilerFlags *flags); +extern int _PyObject_SupportedAsScript(PyObject *); extern const char* _Py_SourceAsString( PyObject *cmd, const char *funcname, diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index d64ac45d85f396..733a15b25f6894 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -1,4 +1,32 @@ +def simple_script(): + assert True + + +def complex_script(): + obj = 'a string' + pickle = __import__('pickle') + def spam_minimal(): + pass + spam_minimal() + data = pickle.dumps(obj) + res = pickle.loads(data) + assert res == obj, (res, obj) + + +def script_with_globals(): + obj1, obj2 = spam(42) + assert obj1 == 42 + assert obj2 is None + + +def script_with_explicit_empty_return(): + return None + + +def script_with_return(): + return True + def spam_minimal(): # no arg defaults or kwarg defaults @@ -141,6 +169,11 @@ def ham_C_closure(z): TOP_FUNCTIONS = [ # shallow + simple_script, + complex_script, + script_with_globals, + script_with_explicit_empty_return, + script_with_return, spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, @@ -179,6 +212,10 @@ def ham_C_closure(z): ] STATELESS_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + script_with_return, spam, spam_minimal, spam_with_builtins, @@ -200,10 +237,26 @@ def ham_C_closure(z): ] STATELESS_CODE = [ *STATELESS_FUNCTIONS, + script_with_globals, spam_with_globals_and_builtins, spam_full, ] +PURE_SCRIPT_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + spam_minimal, + spam_with_builtins, + spam_with_inner_not_closure, + spam_with_inner_closure, +] +SCRIPT_FUNCTIONS = [ + *PURE_SCRIPT_FUNCTIONS, + script_with_globals, + spam_with_globals_and_builtins, +] + # generators diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 6715ee051336a1..32cf8aacaf6b72 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -673,6 +673,20 @@ def test_local_kinds(self): VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW funcs = { + defs.simple_script: {}, + defs.complex_script: { + 'obj': CO_FAST_LOCAL, + 'pickle': CO_FAST_LOCAL, + 'spam_minimal': CO_FAST_LOCAL, + 'data': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.script_with_globals: { + 'obj1': CO_FAST_LOCAL, + 'obj2': CO_FAST_LOCAL, + }, + defs.script_with_explicit_empty_return: {}, + defs.script_with_return: {}, defs.spam_minimal: {}, defs.spam_with_builtins: { 'x': CO_FAST_LOCAL, @@ -898,6 +912,19 @@ def new_var_counts(*, } funcs = { + defs.simple_script: new_var_counts(), + defs.complex_script: new_var_counts( + purelocals=5, + globalvars=1, + attrs=2, + ), + defs.script_with_globals: new_var_counts( + purelocals=2, + globalvars=1, + ), + defs.script_with_explicit_empty_return: new_var_counts(), + defs.script_with_return: new_var_counts(), + defs.spam_minimal: new_var_counts(), defs.spam_minimal: new_var_counts(), defs.spam_with_builtins: new_var_counts( purelocals=4, diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 5ac0080db435a8..b366a29645e9f2 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -758,6 +758,126 @@ def test_other_objects(self): ]) +class PureShareableScriptTests(_GetXIDataTests): + + MODE = 'script-pure' + + VALID_SCRIPTS = [ + '', + 'spam', + '# a comment', + 'print("spam")', + 'raise Exception("spam")', + """if True: + do_something() + """, + """if True: + def spam(x): + return x + class Spam: + def eggs(self): + return 42 + x = Spam().eggs() + raise ValueError(spam(x)) + """, + ] + INVALID_SCRIPTS = [ + ' pass', # IndentationError + '----', # SyntaxError + """if True: + def spam(): + # no body + spam() + """, # IndentationError + ] + + def test_valid_str(self): + self.assert_roundtrip_not_equal([ + *self.VALID_SCRIPTS, + ], expecttype=types.CodeType) + + def test_invalid_str(self): + self.assert_not_shareable([ + *self.INVALID_SCRIPTS, + ]) + + def test_valid_bytes(self): + self.assert_roundtrip_not_equal([ + *(s.encode('utf8') for s in self.VALID_SCRIPTS), + ], expecttype=types.CodeType) + + def test_invalid_bytes(self): + self.assert_not_shareable([ + *(s.encode('utf8') for s in self.INVALID_SCRIPTS), + ]) + + def test_pure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + ]) + + def test_pure_script_function(self): + self.assert_roundtrip_not_equal([ + *defs.PURE_SCRIPT_FUNCTIONS, + ], expecttype=types.CodeType) + + def test_impure_script_function(self): + self.assert_not_shareable([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_function(self): + self.assert_not_shareable([ + *(f for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *defs.FUNCTION_LIKE, + ]) + + def test_other_objects(self): + self.assert_not_shareable([ + None, + True, + False, + Ellipsis, + NotImplemented, + (), + [], + {}, + object(), + ]) + + +class ShareableScriptTests(PureShareableScriptTests): + + MODE = 'script' + + def test_impure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_function(self): + self.assert_roundtrip_not_equal([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ], expecttype=types.CodeType) + + class ShareableTypeTests(_GetXIDataTests): MODE = 'xidata' diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 63f1d079d8d312..3030f45d72cefa 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1989,6 +1989,16 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) goto error; } } + else if (strcmp(mode, "script") == 0) { + if (_PyCode_GetScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script-pure") == 0) { + if (_PyCode_GetPureScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } else { PyErr_Format(PyExc_ValueError, "unsupported mode %R", modeobj); goto error; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 74ce02f1a26401..7d7e6551c3f2d2 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -6,8 +6,10 @@ #include "osdefs.h" // MAXPATHLEN #include "pycore_ceval.h" // _Py_simple_func #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_function.h" // _PyFunction_VerifyStateless() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() @@ -784,6 +786,131 @@ _PyMarshal_GetXIData(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) } +/* script wrapper */ + +static int +verify_script(PyThreadState *tstate, PyCodeObject *co, int checked, int pure) +{ + // Make sure it isn't a closure and (optionally) doesn't use globals. + PyObject *builtins = NULL; + if (pure) { + builtins = _PyEval_GetBuiltins(tstate); + assert(builtins != NULL); + } + if (checked) { + assert(_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) == 0); + } + else if (_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) < 0) { + return -1; + } + // Make sure it doesn't have args. + if (co->co_argcount > 0 + || co->co_posonlyargcount > 0 + || co->co_kwonlyargcount > 0 + || co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) + { + _PyErr_SetString(tstate, PyExc_ValueError, + "code with args not supported"); + return -1; + } + // Make sure it doesn't return anything. + if (!_PyCode_ReturnsOnlyNone(co)) { + _PyErr_SetString(tstate, PyExc_ValueError, + "code that returns a value is not a script"); + return -1; + } + return 0; +} + +static int +get_script_xidata(PyThreadState *tstate, PyObject *obj, int pure, + _PyXIData_t *xidata) +{ + // Get the corresponding code object. + PyObject *code = NULL; + int checked = 0; + if (PyCode_Check(obj)) { + code = obj; + Py_INCREF(code); + } + else if (PyFunction_Check(obj)) { + code = PyFunction_GET_CODE(obj); + assert(code != NULL); + Py_INCREF(code); + if (pure) { + if (_PyFunction_VerifyStateless(tstate, obj) < 0) { + goto error; + } + checked = 1; + } + } + else { + const char *filename = " - {% endif %} - - {% if builder != "htmlhelp" %} + {% endif %} + {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} From 4adb1ddac8860131c86898ddaa3d31a77b98bfce Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 14 May 2025 20:18:04 +0200 Subject: [PATCH 098/713] [3.14] Improve tests for str to Fraction conversion (GH-134010) (GH-134016) (cherry picked from commit 17d0fec702e43e27a9c3f37f8abe06a19b656bed) Co-authored-by: Serhiy Storchaka --- Lib/test/test_fractions.py | 207 +++++++++++++++++-------------------- 1 file changed, 97 insertions(+), 110 deletions(-) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 84faa63606439e..37de3ad380e633 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1,7 +1,7 @@ """Tests for Lib/fractions.py.""" from decimal import Decimal -from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, adjust_int_max_str_digits import math import numbers import operator @@ -395,12 +395,14 @@ class B(metaclass=M): def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((5, 1), _components(F("005"))) self.assertEqual((3, 2), _components(F("3/2"))) self.assertEqual((3, 2), _components(F("3 / 2"))) self.assertEqual((3, 2), _components(F(" \n +3/2"))) self.assertEqual((-3, 2), _components(F("-3/2 "))) - self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((13, 2), _components(F(" 0013/002 \n "))) self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((16, 5), _components(F("003.2"))) self.assertEqual((-16, 5), _components(F(" -3.2 "))) self.assertEqual((-3, 1), _components(F(" -3. "))) self.assertEqual((3, 5), _components(F(" .6 "))) @@ -419,116 +421,101 @@ def testFromString(self): self.assertRaisesMessage( ZeroDivisionError, "Fraction(3, 0)", F, "3/0") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '3/'", - F, "3/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/2'", - F, "/2") - self.assertRaisesMessage( - # Denominators don't need a sign. - ValueError, "Invalid literal for Fraction: '3/+2'", - F, "3/+2") - self.assertRaisesMessage( - # Imitate float's parsing. - ValueError, "Invalid literal for Fraction: '+ 3/2'", - F, "+ 3/2") - self.assertRaisesMessage( - # Avoid treating '.' as a regex special character. - ValueError, "Invalid literal for Fraction: '3a2'", - F, "3a2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3/7.2'", - F, "3/7.2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3.2/7'", - F, "3.2/7") - self.assertRaisesMessage( - # Allow 3. and .3, but not . - ValueError, "Invalid literal for Fraction: '.'", - F, ".") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_'", - F, "_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1'", - F, "_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2'", - F, "1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/_'", - F, "/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1_/'", - F, "1_/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1/'", - F, "_1/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2/'", - F, "1__2/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_'", - F, "1/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_1'", - F, "1/_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/1__2'", - F, "1/1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1._111'", - F, "1._111") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1__1'", - F, "1.1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+_1'", - F, "1.1e+_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+1__1'", - F, "1.1e+1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.dd'", - F, "123.dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.5_dd'", - F, "123.5_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: 'dd.5'", - F, "dd.5") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '7_dd'", - F, "7_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/dd'", - F, "1/dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/123_dd'", - F, "1/123_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789edd'", - F, "789edd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789e2_dd'", - F, "789e2_dd") + + def check_invalid(s): + msg = "Invalid literal for Fraction: " + repr(s) + self.assertRaisesMessage(ValueError, msg, F, s) + + check_invalid("3/") + check_invalid("/2") + # Denominators don't need a sign. + check_invalid("3/+2") + check_invalid("3/-2") + # Imitate float's parsing. + check_invalid("+ 3/2") + check_invalid("- 3/2") + # Avoid treating '.' as a regex special character. + check_invalid("3a2") + # Don't accept combinations of decimals and rationals. + check_invalid("3/7.2") + check_invalid("3.2/7") + # No space around dot. + check_invalid("3 .2") + check_invalid("3. 2") + # No space around e. + check_invalid("3.2 e1") + check_invalid("3.2e 1") + # Fractional part don't need a sign. + check_invalid("3.+2") + check_invalid("3.-2") + # Only accept base 10. + check_invalid("0x10") + check_invalid("0x10/1") + check_invalid("1/0x10") + check_invalid("0x10.") + check_invalid("0x10.1") + check_invalid("1.0x10") + check_invalid("1.0e0x10") + # Only accept decimal digits. + check_invalid("³") + check_invalid("³/2") + check_invalid("3/²") + check_invalid("³.2") + check_invalid("3.²") + check_invalid("3.2e²") + check_invalid("¼") + # Allow 3. and .3, but not . + check_invalid(".") + check_invalid("_") + check_invalid("_1") + check_invalid("1__2") + check_invalid("/_") + check_invalid("1_/") + check_invalid("_1/") + check_invalid("1__2/") + check_invalid("1/_") + check_invalid("1/_1") + check_invalid("1/1__2") + check_invalid("1._111") + check_invalid("1.1__1") + check_invalid("1.1e+_1") + check_invalid("1.1e+1__1") + check_invalid("123.dd") + check_invalid("123.5_dd") + check_invalid("dd.5") + check_invalid("7_dd") + check_invalid("1/dd") + check_invalid("1/123_dd") + check_invalid("789edd") + check_invalid("789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '" + val + "'", - F, val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/" + val + "'", - F, "1/" + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1." + val + "'", - F, "1." + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1+e" + val + "'", - F, "1.1+e" + val) + check_invalid(val) + check_invalid("1/" + val) + check_invalid("1." + val) + check_invalid("." + val) + check_invalid("1.1+e" + val) + check_invalid("1.1e" + val) + + def test_limit_int(self): + maxdigits = 5000 + with adjust_int_max_str_digits(maxdigits): + val = '1' * maxdigits + num = (10**maxdigits - 1)//9 + self.assertEqual((num, 1), _components(F(val))) + self.assertRaises(ValueError, F, val + '1') + self.assertEqual((num, 2), _components(F(val + '/2'))) + self.assertRaises(ValueError, F, val + '1/2') + self.assertEqual((1, num), _components(F('1/' + val))) + self.assertRaises(ValueError, F, '1/1' + val) + self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), + _components(F('1.' + val))) + self.assertRaises(ValueError, F, '1.1' + val) + self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) + self.assertRaises(ValueError, F, '.1' + val) + self.assertRaises(ValueError, F, '1.1e1' + val) + self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) + self.assertRaises(ValueError, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) From b6299e8f16c1e6ab1d3dab42f9803e00fb0bdddb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 03:32:45 +0200 Subject: [PATCH 099/713] [3.14] gh-133986: Document string split algorithm when sep is None and maxsplit is 0 (GH-133987) (#133993) --------- (cherry picked from commit 3e23047363f384b7254b7af51afe4e353be94167) Co-authored-by: Joey Smith Co-authored-by: Ned Batchelder Co-authored-by: Semyon Moroz --- Doc/library/stdtypes.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 1d9a655c7664ea..3486a18b5cb1f0 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2269,6 +2269,18 @@ expression support in the :mod:`re` module). >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only + leading runs of consecutive whitespace are considered. + + For example:: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + .. index:: single: universal newlines; str.splitlines method From f2a4b1066be66574a147bc353efc91a6dc65753d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 08:39:39 +0200 Subject: [PATCH 100/713] [3.14] gh-133970: Make PEP750 types generic (GH-133976) (#134029) gh-133970: Make PEP750 types generic (GH-133976) (cherry picked from commit c3a1da5b9397867e6b3169dd17cb33ef2898da4a) Co-authored-by: sobolevn --- Lib/test/test_genericalias.py | 6 +++++- .../Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst | 2 ++ Objects/interpolationobject.c | 2 ++ Objects/templateobject.c | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 8d21ded45014ba..ea0dc241e39475 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -61,6 +61,7 @@ from tkinter import Event except ImportError: Event = None +from string.templatelib import Template, Interpolation from typing import TypeVar T = TypeVar('T') @@ -139,7 +140,10 @@ class BaseTest(unittest.TestCase): DictReader, DictWriter, array, staticmethod, - classmethod] + classmethod, + Template, + Interpolation, + ] if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader, ctypes.py_object)) if ValueProxy is not None: diff --git a/Misc/NEWS.d/next/Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst b/Misc/NEWS.d/next/Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst new file mode 100644 index 00000000000000..ddf456d3939056 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst @@ -0,0 +1,2 @@ +Make :class:`!string.templatelib.Template` and +:class:`!string.templatelib.Interpolation` generic. diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index aaea3b8c0670c9..a5d407a7b0e296 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -137,6 +137,8 @@ interpolation_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef interpolation_methods[] = { {"__reduce__", interpolation_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL}, }; diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 7d356980b56cbb..06cb19e0b6d056 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -444,6 +444,8 @@ template_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef template_methods[] = { {"__reduce__", template_reduce, METH_NOARGS, NULL}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, {NULL, NULL}, }; From fede4ed1d34242b9354e202727a7456562f2294d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 09:47:24 +0200 Subject: [PATCH 101/713] [3.14] Test also error messages in test_limit_int. (GH-134018) (GH-134033) (cherry picked from commit e123a1d09bcb75aae0c53d9d732de5d9a1252efe) Co-authored-by: Serhiy Storchaka --- Lib/test/test_fractions.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 37de3ad380e633..96b3f30519459b 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -500,22 +500,23 @@ def check_invalid(s): def test_limit_int(self): maxdigits = 5000 with adjust_int_max_str_digits(maxdigits): + msg = 'Exceeds the limit' val = '1' * maxdigits num = (10**maxdigits - 1)//9 self.assertEqual((num, 1), _components(F(val))) - self.assertRaises(ValueError, F, val + '1') + self.assertRaisesRegex(ValueError, msg, F, val + '1') self.assertEqual((num, 2), _components(F(val + '/2'))) - self.assertRaises(ValueError, F, val + '1/2') + self.assertRaisesRegex(ValueError, msg, F, val + '1/2') self.assertEqual((1, num), _components(F('1/' + val))) - self.assertRaises(ValueError, F, '1/1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1/1' + val) self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), _components(F('1.' + val))) - self.assertRaises(ValueError, F, '1.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1' + val) self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) - self.assertRaises(ValueError, F, '.1' + val) - self.assertRaises(ValueError, F, '1.1e1' + val) + self.assertRaisesRegex(ValueError, msg, F, '.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1e1' + val) self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) - self.assertRaises(ValueError, F, '1.1e' + '0' * (maxdigits+1)) + self.assertRaisesRegex(ValueError, msg, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) From 620065c33e1176a1074e2dd2b311c02d6b04ea42 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 11:39:20 +0200 Subject: [PATCH 102/713] [3.14] gh-133403: Check `Tools/build/deepfreeze.py` with mypy (GH-133802) (#134038) gh-133403: Check `Tools/build/deepfreeze.py` with mypy (GH-133802) (cherry picked from commit 7eaa09739059aaac4812395f8d6bb586af8eadcc) Co-authored-by: sobolevn --- .github/workflows/mypy.yml | 2 ++ Tools/build/.ruff.toml | 2 +- Tools/build/deepfreeze.py | 51 ++++++++++++++++++++++++-------------- Tools/build/mypy.ini | 4 ++- Tools/build/umarshal.py | 11 ++++---- 5 files changed, 45 insertions(+), 25 deletions(-) diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 2c7c4945b8b8c2..9dbdd606fff472 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -14,10 +14,12 @@ on: - "Lib/tomllib/**" - "Misc/mypy/**" - "Tools/build/compute-changes.py" + - "Tools/build/deepfreeze.py" - "Tools/build/generate_sbom.py" - "Tools/build/generate-build-details.py" - "Tools/build/verify_ensurepip_wheels.py" - "Tools/build/update_file.py" + - "Tools/build/umarshal.py" - "Tools/cases_generator/**" - "Tools/clinic/**" - "Tools/jit/**" diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index fa7689d45dbcb7..dcbf2936290f12 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -1,7 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings [per-file-target-version] -"deepfreeze.py" = "py310" +"deepfreeze.py" = "py311" # requires `code.co_exceptiontable` "stable_abi.py" = "py311" # requires 'tomllib' [format] diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 23f58447937976..2b9f03aebb6d7e 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -2,9 +2,12 @@ The script may be executed by _bootstrap_python interpreter. Shared library extension modules are not available in that case. -On Windows, and in cross-compilation cases, it is executed -by Python 3.10, and 3.11 features are not available. +Requires 3.11+ to be executed, +because relies on `code.co_qualname` and `code.co_exceptiontable`. """ + +from __future__ import annotations + import argparse import builtins import collections @@ -13,10 +16,14 @@ import re import time import types -from typing import TextIO import umarshal +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any, TextIO + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False @@ -45,8 +52,8 @@ def make_string_literal(b: bytes) -> str: next_code_version = 1 -def get_localsplus(code: types.CodeType): - a = collections.defaultdict(int) +def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]: + a: collections.defaultdict[str, int] = collections.defaultdict(int) for name in code.co_varnames: a[name] |= CO_FAST_LOCAL for name in code.co_cellvars: @@ -136,7 +143,7 @@ def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: return identifiers, strings @contextlib.contextmanager - def indent(self) -> None: + def indent(self) -> Iterator[None]: save_level = self.level try: self.level += 1 @@ -148,7 +155,7 @@ def write(self, arg: str) -> None: self.file.writelines((" "*self.level, arg, "\n")) @contextlib.contextmanager - def block(self, prefix: str, suffix: str = "") -> None: + def block(self, prefix: str, suffix: str = "") -> Iterator[None]: self.write(prefix + " {") with self.indent(): yield @@ -250,9 +257,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str: co_names = self.generate(name + "_names", code.co_names) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) - co_qualname = self.generate(name + "_qualname", code.co_qualname) co_linetable = self.generate(name + "_linetable", code.co_linetable) - co_exceptiontable = self.generate(name + "_exceptiontable", code.co_exceptiontable) + # We use 3.10 for type checking, but this module requires 3.11 + # TODO: bump python version for this script. + co_qualname = self.generate( + name + "_qualname", + code.co_qualname, # type: ignore[attr-defined] + ) + co_exceptiontable = self.generate( + name + "_exceptiontable", + code.co_exceptiontable, # type: ignore[attr-defined] + ) # These fields are not directly accessible localsplusnames, localspluskinds = get_localsplus(code) co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames) @@ -379,13 +394,13 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: + def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str: try: - fs = sorted(fs) + fs_sorted = sorted(fs) except TypeError: # frozen set with incompatible types, fallback to repr() - fs = sorted(fs, key=repr) - ret = self.generate_tuple(name, tuple(fs)) + fs_sorted = sorted(fs, key=repr) + ret = self.generate_tuple(name, tuple(fs_sorted)) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -402,7 +417,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, types.CodeType) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -458,7 +473,7 @@ def decode_frozen_data(source: str) -> types.CodeType: if re.match(FROZEN_DATA_LINE, line): values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) - return umarshal.loads(data) + return umarshal.loads(data) # type: ignore[no-any-return] def generate(args: list[str], output: TextIO) -> None: @@ -494,12 +509,12 @@ def generate(args: list[str], output: TextIO) -> None: help="Input file and module name (required) in file:modname format") @contextlib.contextmanager -def report_time(label: str): - t0 = time.time() +def report_time(label: str) -> Iterator[None]: + t0 = time.perf_counter() try: yield finally: - t1 = time.time() + t1 = time.perf_counter() if verbose: print(f"{label}: {t1-t0:.3f} sec") diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 62d7e150fd8521..123dc895f90a1f 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -4,10 +4,12 @@ # .github/workflows/mypy.yml files = Tools/build/compute-changes.py, + Tools/build/deepfreeze.py, Tools/build/generate-build-details.py, Tools/build/generate_sbom.py, Tools/build/verify_ensurepip_wheels.py, - Tools/build/update_file.py + Tools/build/update_file.py, + Tools/build/umarshal.py pretty = True diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index 679fa7caf9f931..865cffc2440122 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -145,12 +145,12 @@ def r_PyLong(self) -> int: def r_float_bin(self) -> float: buf = self.r_string(8) import struct # Lazy import to avoid breaking UNIX build - return struct.unpack("d", buf)[0] + return struct.unpack("d", buf)[0] # type: ignore[no-any-return] def r_float_str(self) -> float: n = self.r_byte() buf = self.r_string(n) - return ast.literal_eval(buf.decode("ascii")) + return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return] def r_ref_reserve(self, flag: int) -> int: if flag: @@ -306,7 +306,7 @@ def loads(data: bytes) -> Any: return r.r_object() -def main(): +def main() -> None: # Test import marshal import pprint @@ -314,8 +314,9 @@ def main(): data = marshal.dumps(sample) retval = loads(data) assert retval == sample, retval - sample = main.__code__ - data = marshal.dumps(sample) + + sample2 = main.__code__ + data = marshal.dumps(sample2) retval = loads(data) assert isinstance(retval, Code), retval pprint.pprint(retval.__dict__) From 29d69e8b661e5d4d270f44c03a2c678cc3ebc58d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 15:34:50 +0200 Subject: [PATCH 103/713] [3.14] gh-77065: add missing parameter `echo_char` in `getpass.fallback_getpass` (GH-133849) (#134053) gh-77065: add missing parameter `echo_char` in `getpass.fallback_getpass` (GH-133849) (cherry picked from commit d029a1a1cba515510f31c9e88b7c2a6b7ebee2a7) Co-authored-by: Semyon Moroz --- Lib/getpass.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/getpass.py b/Lib/getpass.py index f571425e54178a..a4dbd2ea6502d4 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -132,14 +132,15 @@ def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): return pw -def fallback_getpass(prompt='Password: ', stream=None): +def fallback_getpass(prompt='Password: ', stream=None, *, echo_char=None): + _check_echo_char(echo_char) import warnings warnings.warn("Can not control echo on the terminal.", GetPassWarning, stacklevel=2) if not stream: stream = sys.stderr print("Warning: Password input may be echoed.", file=stream) - return _raw_input(prompt, stream) + return _raw_input(prompt, stream, echo_char=echo_char) def _check_echo_char(echo_char): From 31deafb8eee8c07a084c77e57101d99a2b993698 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 15:37:41 +0200 Subject: [PATCH 104/713] [3.14] gh-133410: Fix PR detection in build workflow (GH-133671) (#134055) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/reusable-context.yml | 1 + Tools/build/compute-changes.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index 73dc254edc5fbc..d2668ddcac1a3d 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -97,6 +97,7 @@ jobs: run: python Tools/build/compute-changes.py env: GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index cfdd55fd1925fd..b5993d29b92972 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -57,7 +57,7 @@ class Outputs: def compute_changes() -> None: target_branch, head_ref = git_refs() - if target_branch and head_ref: + if os.environ.get("GITHUB_EVENT_NAME", "") == "pull_request": # Getting changed files only makes sense on a pull request files = get_changed_files(target_branch, head_ref) outputs = process_changed_files(files) From 438f464a5a402ad6901578469dbca7b67c3e11dc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 17:41:37 +0200 Subject: [PATCH 105/713] [3.14] gh-77065: Use `putwch` instead of `putch` in `getpass.win_getpass` (GH-134058) (#134059) gh-77065: Use `putwch` instead of `putch` in `getpass.win_getpass` (GH-134058) (cherry picked from commit 52a7a22a6b85a470e993b3399b0ee043e5c3596e) Co-authored-by: Semyon Moroz --- Lib/getpass.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/getpass.py b/Lib/getpass.py index a4dbd2ea6502d4..1dd40e25e09068 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -119,9 +119,9 @@ def win_getpass(prompt='Password: ', stream=None, *, echo_char=None): raise KeyboardInterrupt if c == '\b': if echo_char and pw: - msvcrt.putch('\b') - msvcrt.putch(' ') - msvcrt.putch('\b') + msvcrt.putwch('\b') + msvcrt.putwch(' ') + msvcrt.putwch('\b') pw = pw[:-1] else: pw = pw + c From ba06ccbb6dc207526ff09a6fec5cac424acfc62a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 15 May 2025 23:18:26 +0200 Subject: [PATCH 106/713] Improve the administrative install docs for Python Install Manager (GH-134066) (cherry picked from commit 6a2296329117463fd09abc73656f1d7b48076100) Co-authored-by: Steve Dower --- Doc/using/windows.rst | 46 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 38b97a34fcda31..cd22148b531b84 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -98,8 +98,9 @@ does not affect any installs of Python runtimes. Uninstalling the Python install manager does not uninstall any Python runtimes. If you are not able to install an MSIX in your context, for example, you are -using automated deployment software that does not support it, please see -:ref:`pymanager-advancedinstall` below for more information. +using automated deployment software that does not support it, or are targeting +Windows Server 2019, please see :ref:`pymanager-advancedinstall` below for more +information. Basic Use @@ -515,6 +516,11 @@ per-machine installs to its default location in Program Files. It will attempt to modify the system :envvar:`PATH` environment variable to include this install location, but be sure to validate this on your configuration. +.. note:: + + Windows Server 2019 is the only version of Windows that CPython supports that + does not support MSIX. For Windows Server 2019, you should use the MSI. + Be aware that the MSI package does not bundle any runtimes, and so is not suitable for installs into offline environments without also creating an offline install index. See :ref:`pymanager-offline` and :ref:`pymanager-admin-config` @@ -535,13 +541,32 @@ depending on whether it was installed from python.org or through the Windows Store. Attempting to run the executable directly from Program Files is not recommended. -To programmatically install or uninstall the MSIX without using your -distribution platform's native support, the `Add-AppxPackage -`_ and -`Remove-AppxPackage `_ -PowerShell cmdlets are simplest to use: +To programmatically install the Python install manager, it is easiest to use +WinGet, which is included with all supported versions of Windows: -.. code:: +.. code-block:: powershell + + $> winget install 9NQ7512CXL7T -e --accept-package-agreements --disable-interactivity + + # Optionally run the configuration checker and accept all changes + $> py install --configure -y + +To download the Python install manager and install on another machine, the +following WinGet command will download the required files from the Store to your +Downloads directory (add ``-d `` to customize the output location). +This also generates a YAML file that appears to be unnecessary, as the +downloaded MSIX can be installed by launching or using the commands below. + +.. code-block:: powershell + + $> winget download 9NQ7512CXL7T -e --skip-license --accept-package-agreements --disable-interactivity + +To programmatically install or uninstall an MSIX using only PowerShell, the +`Add-AppxPackage `_ +and `Remove-AppxPackage `_ +PowerShell cmdlets are recommended: + +.. code-block:: powershell $> Add-AppxPackage C:\Downloads\python-manager-25.0.msix ... @@ -555,6 +580,11 @@ to install the Python install manager for all users from the MSIX package. Users will still need to install their own copies of Python itself, as there is no way to trigger those installs without being a logged in user. +Note that the MSIX downloadable from the Store and from the Python website are +subtly different and cannot be installed at the same time. Wherever possible, +we suggest using the above commands to download the package from the Store to +reduce the risk of setting up conflicting installs. + .. _pymanager-admin-config: Administrative Configuration From 5374762234389e93c6ec305371e91f7b6e09e8ca Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 12:04:24 +0200 Subject: [PATCH 107/713] [3.14] gh-130000: Release the GIL in winreg when doing Windows API calls (GH-130001) (#134072) gh-130000: Release the GIL in winreg when doing Windows API calls (GH-130001) (cherry picked from commit 7a504b3d5da98874536834481539c19ba4a265af) Co-authored-by: AN Long --- PC/winreg.c | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) diff --git a/PC/winreg.c b/PC/winreg.c index c0de5c1353a349..c342be92b98f08 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -426,7 +426,9 @@ PyHKEY_Close(winreg_state *st, PyObject *ob_handle) if (PyHKEY_Check(st, ob_handle)) { ((PyHKEYObject*)ob_handle)->hkey = 0; } + Py_BEGIN_ALLOW_THREADS rc = key ? RegCloseKey(key) : ERROR_SUCCESS; + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); return rc == ERROR_SUCCESS; @@ -499,14 +501,21 @@ PyWinObject_CloseHKEY(winreg_state *st, PyObject *obHandle) } #if SIZEOF_LONG >= SIZEOF_HKEY else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); + long rc; + Py_BEGIN_ALLOW_THREADS + rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); + Py_END_ALLOW_THREADS ok = (rc == ERROR_SUCCESS); if (!ok) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); } #else else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsVoidPtr(obHandle)); + long rc; + HKEY hkey = (HKEY)PyLong_AsVoidPtr(obHandle); + Py_BEGIN_ALLOW_THREADS + rc = RegCloseKey(hkey); + Py_END_ALLOW_THREADS ok = (rc == ERROR_SUCCESS); if (!ok) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); @@ -924,7 +933,9 @@ winreg_CreateKey_impl(PyObject *module, HKEY key, const wchar_t *sub_key) (Py_ssize_t)KEY_WRITE) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyW(key, sub_key, &retKey); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKey"); return NULL; @@ -973,8 +984,10 @@ winreg_CreateKeyEx_impl(PyObject *module, HKEY key, const wchar_t *sub_key, (Py_ssize_t)access) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyExW(key, sub_key, reserved, NULL, 0, access, NULL, &retKey, NULL); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKeyEx"); return NULL; @@ -1187,10 +1200,12 @@ winreg_EnumValue_impl(PyObject *module, HKEY key, int index) (Py_ssize_t)key, index) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, - &retValueSize, &retDataSize, NULL, NULL)) - != ERROR_SUCCESS) + + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + &retValueSize, &retDataSize, NULL, NULL); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); ++retValueSize; /* include null terminators */ @@ -1477,9 +1492,11 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) if (PySys_Audit("winreg.QueryInfoKey", "n", (Py_ssize_t)key) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, - &nValues, NULL, NULL, NULL, &ft)) - != ERROR_SUCCESS) { + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, + &nValues, NULL, NULL, NULL, &ft); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) { return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); } li.LowPart = ft.dwLowDateTime; @@ -1587,7 +1604,9 @@ winreg_QueryValue_impl(PyObject *module, HKEY key, const wchar_t *sub_key) PyMem_Free(pbuf); } if (childKey != key) { + Py_BEGIN_ALLOW_THREADS RegCloseKey(childKey); + Py_END_ALLOW_THREADS } return result; } @@ -1625,7 +1644,9 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) (Py_ssize_t)key, NULL, name) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, NULL, NULL, &bufSize); + Py_END_ALLOW_THREADS if (rc == ERROR_MORE_DATA) bufSize = 256; else if (rc != ERROR_SUCCESS) @@ -1637,8 +1658,10 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) while (1) { retSize = bufSize; + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, &typ, (BYTE *)retBuf, &retSize); + Py_END_ALLOW_THREADS if (rc != ERROR_MORE_DATA) break; From 7725ba84619330e071bf0d91a3e4f0d61d17650d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 12:49:08 +0200 Subject: [PATCH 108/713] [3.14] gh-134069: bump HACL* revision to incoporate `memset_s` (GH-134027) (#134084) Bumps the HACL* revision to include recent revisions that corrects issues building with legacy/cross-platform macOS SDKs. (cherry picked from commit 1566c34dc76ec6139e6827fbab6d76e084a63d9d) Signed-off-by: aeiouaeiouaeiouaeiouaeiouaeiou Co-authored-by: aeiouaeiouaeiouaeiouaeiouaeiou --- Misc/sbom.spdx.json | 52 +- Modules/_hacl/Hacl_Hash_Blake2b.c | 52 +- Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c | 52 +- Modules/_hacl/Hacl_Hash_Blake2s.c | 52 +- Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c | 52 +- Modules/_hacl/Hacl_Hash_MD5.c | 523 +++++++----------- Modules/_hacl/Hacl_Hash_SHA1.c | 12 +- Modules/_hacl/Hacl_Hash_SHA2.c | 42 +- Modules/_hacl/Hacl_Hash_SHA3.c | 22 +- Modules/_hacl/Hacl_Streaming_HMAC.c | 127 ++--- Modules/_hacl/Lib_Memzero0.c | 20 +- .../include/krml/FStar_UInt128_Verified.h | 45 +- Modules/_hacl/refresh.sh | 2 +- 13 files changed, 405 insertions(+), 648 deletions(-) diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 4a697d047ca6e4..10a5c646d2a59e 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -314,11 +314,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4b6e7696e8d84f322fb24b1fbb08ccb9b0e7d51b" + "checksumValue": "808af7ff8a2cb2b4ef3a9ce3dbfef58d90828c9f" }, { "algorithm": "SHA256", - "checksumValue": "50a65a34a7a7569eedf7fa864a7892eeee5840a7fdf6fa8f1e87d42c65f6c877" + "checksumValue": "6a492aa586f2d10b1b300ce8ce4c72c976ff7548fee667aded2253f99969ac87" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" @@ -342,11 +342,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0f75e44a42775247a46acc2beaa6bae8f199a3d9" + "checksumValue": "1bb072f2be9e5d194274fdcc87825cb094e4b32e" }, { "algorithm": "SHA256", - "checksumValue": "03b612c24193464ed6848aeebbf44f9266b78ec6eed2486056211cde8992c49a" + "checksumValue": "9eb22953ce60dde9dc970fec9dfce9d94235f4b7ccd8f0151cad4707dc835d1d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" @@ -384,11 +384,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "65bf44140691b046dcfed3ab1576dbf8bbf96dc5" + "checksumValue": "cdd6e9ca86dbede92d1a47b9224d2af70c599a71" }, { "algorithm": "SHA256", - "checksumValue": "0f98959dafffce039ade9d296f7a05bed151c9c512498f48e4b326a5523a240b" + "checksumValue": "f3204f3e60734d811b6630f879b69ce54eaf367f3fca5889c1026e7a1f3ee1e4" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" @@ -412,11 +412,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0da9782455923aede8d8dce9dfdc38f4fc1de572" + "checksumValue": "4056bb6e3ed184400d1610bdfd4260b3fd05ccbb" }, { "algorithm": "SHA256", - "checksumValue": "2d17ae768fd3d7d6decddd8b4aaf23ce02a809ee62bb98da32c8a7f54acf92d0" + "checksumValue": "c93746df2f219cbb1634ee6fb0ab1c4cbd381d1f36c637114c68346c9935326d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" @@ -454,11 +454,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "38e8d96ef1879480780494058a93cec181f8d6d7" + "checksumValue": "61f678cd9234c6eab5d4409ae66f470b068b959b" }, { "algorithm": "SHA256", - "checksumValue": "61e77d2063cf60c96e9ce06af215efe5d42c43026833bffed5732326fe97ed1e" + "checksumValue": "5b91ed0339074e2e546119833398e2cdb7190c33a59c405bf43b2417c789547d" } ], "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" @@ -482,11 +482,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "986dd5ba0b34d15f3e5e5c656979aea1b502e8aa" + "checksumValue": "c7fc5c9721caf37c5a24c9beff27b0ac2ed68cc9" }, { "algorithm": "SHA256", - "checksumValue": "38d5f1f2e67a0eb30789f81fc56c07a6e7246e2b1be6c65485bcca1dcd0e0806" + "checksumValue": "ce08721d491f3b8a9bd4cde6c27bfcc8fc01471512ccca4bd3c0b764cb551d29" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" @@ -510,11 +510,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f732a6710fe3e13cd28130f0f20504e347d1c412" + "checksumValue": "fffe8c4f67669ac8ccd87c2e0f95db2427481df1" }, { "algorithm": "SHA256", - "checksumValue": "86cf32e4d1f3ba93a94108271923fdafe2204447792a918acf4a2250f352dbde" + "checksumValue": "f8af382de7e29a73c726f9c70770498ddd99e2c4702489ed6e634f0b68597c95" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" @@ -538,11 +538,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "50f75337b31f509b5bfcc7ebb3d066b82a0f1b33" + "checksumValue": "77d3d879dfa5949030bca0e8ee75d3d369ec54a7" }, { "algorithm": "SHA256", - "checksumValue": "c9e1442899e5b902fa39f413f1a3131f7ab5c2283d5100dc8ac675a7d5ebbdf1" + "checksumValue": "8744f5b6e054c3e5c44f413a60f9154b76dd7230135dcee26fd063755ec64be1" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" @@ -566,11 +566,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8140310f505bb2619a749777a91487d666237bcf" + "checksumValue": "417e68ac8498cb2f93a06a19003ea1cc3f0f6753" }, { "algorithm": "SHA256", - "checksumValue": "9d95e6a651c22185d9b7c38f363d30159f810e6fcdc2208f29492837ed891e82" + "checksumValue": "843db4bba78a476d4d53dabe4eed5c9235f490ccc9fdaf19e22af488af858920" } ], "fileName": "Modules/_hacl/Hacl_Streaming_HMAC.c" @@ -608,11 +608,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c9651ef21479c4d8a3b04c5baa1902866dbb1cdf" + "checksumValue": "0a0b7f3714167ad45ddf5a6a48d76f525a119c9c" }, { "algorithm": "SHA256", - "checksumValue": "e039c82ba670606ca111573942baad800f75da467abbc74cd7d1fe175ebcdfaf" + "checksumValue": "135d4afb4812468885c963c9c87a55ba5fae9181df4431af5fbad08588dda229" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -622,11 +622,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "eaa543c778300238dc23034aafeada0951154af1" + "checksumValue": "911c97a0f24067635b164fbca49da76055f192cd" }, { "algorithm": "SHA256", - "checksumValue": "3fd2552d527a23110d61ad2811c774810efb1eaee008f136c2a0d609daa77f5b" + "checksumValue": "972b5111ebada8e11dd60df3119da1af505fd3e0b6c782ead6cab7f1daf134f1" } ], "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" @@ -1752,14 +1752,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "02dfcf0c79d488b120d7f2c2a0f9206301c7927ed5106545e0b6f2aef88da76a" + "checksumValue": "39f6fd4f2fe98aecc995a2fd980fae0e0835cef6e563ebbf25f69d3d3102bafd" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/7720f6d4fc0468a99d5ea6120976bcc271e42727.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/4ef25b547b377dcef855db4289c6a00580e7221c.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:7720f6d4fc0468a99d5ea6120976bcc271e42727:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:4ef25b547b377dcef855db4289c6a00580e7221c:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1767,7 +1767,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "7720f6d4fc0468a99d5ea6120976bcc271e42727" + "versionInfo": "4ef25b547b377dcef855db4289c6a00580e7221c" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c index 21ab2b88c799a6..a5b75d61798949 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -544,11 +544,9 @@ void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn) uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -860,14 +858,10 @@ static Hacl_Hash_Blake2b_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1059,11 +1053,9 @@ static void reset_raw(Hacl_Hash_Blake2b_state_t *state, Hacl_Hash_Blake2b_params uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1200,8 +1192,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1265,8 +1256,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1296,8 +1286,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state10, @@ -1359,8 +1348,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1690,14 +1678,10 @@ Hacl_Hash_Blake2b_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c index c4d9b4a689d28f..f955f1c6115b1e 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -274,11 +274,9 @@ Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t k uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -746,14 +744,10 @@ static Hacl_Hash_Blake2b_Simd256_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -936,11 +930,9 @@ reset_raw(Hacl_Hash_Blake2b_Simd256_state_t *state, Hacl_Hash_Blake2b_params_and uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1075,8 +1067,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1140,8 +1131,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1171,8 +1161,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state10, @@ -1234,8 +1223,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1578,14 +1566,10 @@ Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c index 730ba135afb2fb..0d4fcc7f3951fc 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -543,13 +543,13 @@ void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn) uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -846,16 +846,14 @@ static Hacl_Hash_Blake2s_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1042,13 +1040,13 @@ static void reset_raw(Hacl_Hash_Blake2s_state_t *state, Hacl_Hash_Blake2b_params uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1182,8 +1180,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1237,8 +1234,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1268,8 +1264,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state10, @@ -1321,8 +1316,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1639,16 +1633,14 @@ Hacl_Hash_Blake2s_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c index 7e9cd79544f8f1..fa46be045f3441 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -271,13 +271,13 @@ Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t k uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -736,16 +736,14 @@ static Hacl_Hash_Blake2s_Simd128_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -923,13 +921,13 @@ reset_raw(Hacl_Hash_Blake2s_Simd128_state_t *state, Hacl_Hash_Blake2b_params_and uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1061,8 +1059,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1116,8 +1113,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1147,8 +1143,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state10, @@ -1200,8 +1195,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1531,16 +1525,14 @@ Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 75ce8d2926e6e1..305c75483c06ea 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -66,11 +66,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti0 = _t[0U]; uint32_t v = - vb0 - + - ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << 7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); + vb0 + + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) << 7U | + (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; @@ -82,11 +80,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti1 = _t[1U]; uint32_t v0 = - vb1 - + - ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << 12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); + vb1 + + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) << 12U | + (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; @@ -98,11 +94,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti2 = _t[2U]; uint32_t v1 = - vb2 - + - ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << 17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); + vb2 + + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) << 17U | + (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; @@ -114,11 +108,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti3 = _t[3U]; uint32_t v2 = - vb3 - + - ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << 22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); + vb3 + + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) << 22U | + (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; @@ -130,11 +122,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti4 = _t[4U]; uint32_t v3 = - vb4 - + - ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << 7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); + vb4 + + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) << 7U | + (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; @@ -146,11 +136,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti5 = _t[5U]; uint32_t v4 = - vb5 - + - ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << 12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); + vb5 + + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) << 12U | + (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; @@ -162,11 +150,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti6 = _t[6U]; uint32_t v5 = - vb6 - + - ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << 17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); + vb6 + + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) << 17U | + (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; @@ -178,11 +164,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti7 = _t[7U]; uint32_t v6 = - vb7 - + - ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << 22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); + vb7 + + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) << 22U | + (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; @@ -194,11 +178,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti8 = _t[8U]; uint32_t v7 = - vb8 - + - ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << 7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); + vb8 + + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) << 7U | + (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; @@ -210,11 +192,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti9 = _t[9U]; uint32_t v8 = - vb9 - + - ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << 12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); + vb9 + + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) << 12U | + (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; @@ -226,11 +206,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti10 = _t[10U]; uint32_t v9 = - vb10 - + - ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << 17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); + vb10 + + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) << 17U | + (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; @@ -242,11 +220,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti11 = _t[11U]; uint32_t v10 = - vb11 - + - ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << 22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); + vb11 + + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) << 22U | + (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; @@ -258,11 +234,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti12 = _t[12U]; uint32_t v11 = - vb12 - + - ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << 7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); + vb12 + + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) << 7U | + (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; @@ -274,11 +248,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti13 = _t[13U]; uint32_t v12 = - vb13 - + - ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << 12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); + vb13 + + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) << 12U | + (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; @@ -290,11 +262,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti14 = _t[14U]; uint32_t v13 = - vb14 - + - ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << 17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); + vb14 + + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) << 17U | + (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; @@ -306,11 +276,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti15 = _t[15U]; uint32_t v14 = - vb15 - + - ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << 22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); + vb15 + + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) << 22U | + (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; @@ -322,11 +290,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti16 = _t[16U]; uint32_t v15 = - vb16 - + - ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << 5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); + vb16 + + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) << 5U | + (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; @@ -338,11 +304,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti17 = _t[17U]; uint32_t v16 = - vb17 - + - ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << 9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); + vb17 + + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) << 9U | + (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; @@ -354,11 +318,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti18 = _t[18U]; uint32_t v17 = - vb18 - + - ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << 14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); + vb18 + + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) << 14U | + (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -370,11 +332,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti19 = _t[19U]; uint32_t v18 = - vb19 - + - ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << 20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); + vb19 + + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) << 20U | + (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; @@ -386,11 +346,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti20 = _t[20U]; uint32_t v19 = - vb20 - + - ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << 5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); + vb20 + + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) << 5U | + (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; @@ -402,11 +360,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti21 = _t[21U]; uint32_t v20 = - vb21 - + - ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << 9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); + vb21 + + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) << 9U | + (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; @@ -418,11 +374,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti22 = _t[22U]; uint32_t v21 = - vb22 - + - ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << 14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); + vb22 + + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) << 14U | + (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; @@ -434,11 +388,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti23 = _t[23U]; uint32_t v22 = - vb23 - + - ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << 20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); + vb23 + + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) << 20U | + (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; @@ -450,11 +402,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti24 = _t[24U]; uint32_t v23 = - vb24 - + - ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << 5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); + vb24 + + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) << 5U | + (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; @@ -466,11 +416,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti25 = _t[25U]; uint32_t v24 = - vb25 - + - ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << 9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); + vb25 + + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) << 9U | + (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; @@ -482,11 +430,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti26 = _t[26U]; uint32_t v25 = - vb26 - + - ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << 14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); + vb26 + + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) << 14U | + (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; @@ -498,11 +444,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti27 = _t[27U]; uint32_t v26 = - vb27 - + - ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << 20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); + vb27 + + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) << 20U | + (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; @@ -514,11 +458,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti28 = _t[28U]; uint32_t v27 = - vb28 - + - ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << 5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); + vb28 + + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) << 5U | + (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; @@ -530,11 +472,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti29 = _t[29U]; uint32_t v28 = - vb29 - + - ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << 9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); + vb29 + + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) << 9U | + (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; @@ -546,11 +486,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti30 = _t[30U]; uint32_t v29 = - vb30 - + - ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << 14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); + vb30 + + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) << 14U | + (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; @@ -562,11 +500,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti31 = _t[31U]; uint32_t v30 = - vb31 - + - ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << 20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); + vb31 + + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) << 20U | + (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; @@ -578,11 +514,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti32 = _t[32U]; uint32_t v31 = - vb32 - + - ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << 4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); + vb32 + + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) << 4U | + (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; @@ -594,11 +528,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti33 = _t[33U]; uint32_t v32 = - vb33 - + - ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << 11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); + vb33 + + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) << 11U | + (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; @@ -610,11 +542,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti34 = _t[34U]; uint32_t v33 = - vb34 - + - ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << 16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); + vb34 + + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) << 16U | + (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; @@ -626,11 +556,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti35 = _t[35U]; uint32_t v34 = - vb35 - + - ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << 23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); + vb35 + + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) << 23U | + (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; @@ -642,11 +570,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti36 = _t[36U]; uint32_t v35 = - vb36 - + - ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << 4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); + vb36 + + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) << 4U | + (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; @@ -658,11 +584,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti37 = _t[37U]; uint32_t v36 = - vb37 - + - ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << 11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); + vb37 + + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) << 11U | + (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; @@ -674,11 +598,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti38 = _t[38U]; uint32_t v37 = - vb38 - + - ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << 16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); + vb38 + + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) << 16U | + (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; @@ -690,11 +612,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti39 = _t[39U]; uint32_t v38 = - vb39 - + - ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << 23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); + vb39 + + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) << 23U | + (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; @@ -706,11 +626,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti40 = _t[40U]; uint32_t v39 = - vb40 - + - ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << 4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); + vb40 + + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) << 4U | + (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -722,11 +640,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti41 = _t[41U]; uint32_t v40 = - vb41 - + - ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << 11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); + vb41 + + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) << 11U | + (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; @@ -738,11 +654,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti42 = _t[42U]; uint32_t v41 = - vb42 - + - ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << 16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); + vb42 + + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) << 16U | + (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; @@ -754,11 +668,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti43 = _t[43U]; uint32_t v42 = - vb43 - + - ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << 23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); + vb43 + + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) << 23U | + (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; @@ -770,11 +682,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti44 = _t[44U]; uint32_t v43 = - vb44 - + - ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << 4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); + vb44 + + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) << 4U | + (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; @@ -786,11 +696,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti45 = _t[45U]; uint32_t v44 = - vb45 - + - ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << 11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); + vb45 + + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) << 11U | + (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; @@ -802,11 +710,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti46 = _t[46U]; uint32_t v45 = - vb46 - + - ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << 16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); + vb46 + + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) << 16U | + (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; @@ -818,11 +724,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti47 = _t[47U]; uint32_t v46 = - vb47 - + - ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << 23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); + vb47 + + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) << 23U | + (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -834,11 +738,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti48 = _t[48U]; uint32_t v47 = - vb48 - + - ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << 6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); + vb48 + + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) << 6U | + (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; @@ -850,11 +752,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti49 = _t[49U]; uint32_t v48 = - vb49 - + - ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << 10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); + vb49 + + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) << 10U | + (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; @@ -866,11 +766,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti50 = _t[50U]; uint32_t v49 = - vb50 - + - ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << 15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); + vb50 + + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) << 15U | + (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; @@ -882,11 +780,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti51 = _t[51U]; uint32_t v50 = - vb51 - + - ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << 21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); + vb51 + + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) << 21U | + (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; @@ -898,11 +794,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti52 = _t[52U]; uint32_t v51 = - vb52 - + - ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << 6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); + vb52 + + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) << 6U | + (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; @@ -914,11 +808,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti53 = _t[53U]; uint32_t v52 = - vb53 - + - ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << 10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); + vb53 + + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) << 10U | + (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; @@ -930,11 +822,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti54 = _t[54U]; uint32_t v53 = - vb54 - + - ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << 15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); + vb54 + + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) << 15U | + (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; @@ -946,11 +836,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti55 = _t[55U]; uint32_t v54 = - vb55 - + - ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << 21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); + vb55 + + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) << 21U | + (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; @@ -962,11 +850,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti56 = _t[56U]; uint32_t v55 = - vb56 - + - ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << 6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); + vb56 + + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) << 6U | + (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; @@ -978,11 +864,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti57 = _t[57U]; uint32_t v56 = - vb57 - + - ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << 10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); + vb57 + + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) << 10U | + (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; @@ -994,11 +878,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti58 = _t[58U]; uint32_t v57 = - vb58 - + - ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << 15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); + vb58 + + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) << 15U | + (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; @@ -1010,11 +892,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti59 = _t[59U]; uint32_t v58 = - vb59 - + - ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << 21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); + vb59 + + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) << 21U | + (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; @@ -1026,11 +906,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti60 = _t[60U]; uint32_t v59 = - vb60 - + - ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << 6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); + vb60 + + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) << 6U | + (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; @@ -1042,11 +920,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti61 = _t[61U]; uint32_t v60 = - vb61 - + - ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << 10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); + vb61 + + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) << 10U | + (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; @@ -1058,11 +934,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti62 = _t[62U]; uint32_t v61 = - vb62 - + - ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << 15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); + vb62 + + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) << 15U | + (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; @@ -1074,11 +948,8 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti = _t[63U]; uint32_t v62 = - vb - + - ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << 21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); + vb + + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) << 21U | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1282,8 +1153,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1328,8 +1198,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1359,8 +1228,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -1403,8 +1271,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 508e447bf275da..97bd4f2204cf24 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -315,8 +315,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -361,8 +360,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -392,8 +390,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -436,8 +433,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index d612bafa72cdc4..d2ee0c9ef51721 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -100,15 +100,14 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t k_e_t = k_t; uint32_t t1 = - h02 - + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; uint32_t c1 = b0; @@ -301,15 +300,14 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t k_e_t = k_t; uint64_t t1 = - h02 - + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; uint64_t c1 = b0; @@ -639,8 +637,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -685,8 +682,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -716,8 +712,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -760,8 +755,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1205,8 +1199,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1251,8 +1244,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1282,8 +1274,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state10, @@ -1326,8 +1317,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 87638df9549fbb..466d2b96c0cdfa 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -866,8 +866,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } else if (sz == 0U) @@ -910,8 +909,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -941,8 +939,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, @@ -972,10 +969,8 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(i) - == 0ULL - && (uint64_t)(chunk_len - diff) > 0ULL + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) == 0ULL && + (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); @@ -994,8 +989,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -2422,9 +2416,7 @@ Hacl_Hash_SHA3_shake128_squeeze_nblocks( 5U, 1U, _C[i] = - state[i - + 0U] - ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); + state[i + 0U] ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); KRML_MAYBE_FOR5(i2, 0U, 5U, diff --git a/Modules/_hacl/Hacl_Streaming_HMAC.c b/Modules/_hacl/Hacl_Streaming_HMAC.c index d28b39792af576..8dd7e2c0bf3e71 100644 --- a/Modules/_hacl/Hacl_Streaming_HMAC.c +++ b/Modules/_hacl/Hacl_Streaming_HMAC.c @@ -198,8 +198,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); + st[0U] = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); } if (st == NULL) { @@ -220,8 +219,8 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); + st[0U] = + ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); } if (st == NULL) { @@ -242,8 +241,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_224_a, @@ -270,8 +268,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_256_a, @@ -298,8 +295,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_384_a, @@ -326,8 +322,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_512_a, @@ -354,8 +349,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_224_a, @@ -382,8 +376,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_256_a, @@ -410,8 +403,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_384_a, @@ -438,8 +430,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_512_a, @@ -466,8 +457,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_a, @@ -495,8 +485,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_128_a, @@ -531,8 +520,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_a, @@ -560,8 +548,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_256_a, @@ -2059,8 +2046,8 @@ Hacl_Streaming_HMAC_update( Hacl_Streaming_HMAC_Definitions_index i1 = Hacl_Streaming_HMAC_index_of_state(block_state); if ( - (uint64_t)chunk_len - > max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len + (uint64_t)chunk_len > + max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len ) { return Hacl_Streaming_Types_MaximumLengthExceeded; @@ -2068,9 +2055,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2079,8 +2064,8 @@ Hacl_Streaming_HMAC_update( else { sz = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (chunk_len <= block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - sz) { @@ -2091,9 +2076,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2102,14 +2085,13 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2127,9 +2109,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2138,8 +2118,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2153,8 +2133,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)chunk_len > 0ULL ) @@ -2164,8 +2144,8 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = (chunk_len - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); @@ -2178,8 +2158,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2200,9 +2179,8 @@ Hacl_Streaming_HMAC_update( uint32_t sz10; if ( - total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len10 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == + 0ULL && total_len10 > 0ULL ) { @@ -2211,14 +2189,13 @@ Hacl_Streaming_HMAC_update( else { sz10 = - (uint32_t)(total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len10 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state10, @@ -2233,9 +2210,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2244,8 +2219,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2259,8 +2234,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL ) @@ -2270,13 +2245,12 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = - (chunk_len - diff - ite) - / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); + (chunk_len - diff - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data1_len = n_blocks * block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data2_len = chunk_len - diff - data1_len; @@ -2286,8 +2260,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2324,9 +2297,7 @@ Hacl_Streaming_HMAC_digest( uint32_t r; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2335,8 +2306,8 @@ Hacl_Streaming_HMAC_digest( else { r = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf_1 = buf_; Hacl_Agile_Hash_state_s *s110 = malloc_(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)); diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 28abd1aa4e2d54..f94e0e2254a912 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -11,18 +11,18 @@ #if defined(__APPLE__) && defined(__MACH__) #include // memset_s is available from macOS 10.9, iOS 7, watchOS 2, and on all tvOS and visionOS versions. -# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_9)) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) -# define APPLE_HAS_MEMSET_S 1 +# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_X_VERSION_10_9) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_9)) +# define APPLE_HAS_MEMSET_S +# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_7_0) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_TV) && TARGET_OS_TV) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S +# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && defined(__WATCHOS_2_0) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_VISION) && TARGET_OS_VISION) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S # else -# define APPLE_HAS_MEMSET_S 0 +# undef APPLE_HAS_MEMSET_S # endif #endif @@ -55,7 +55,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { #ifdef _WIN32 SecureZeroMemory(dst, len_); - #elif defined(__APPLE__) && defined(__MACH__) && APPLE_HAS_MEMSET_S + #elif defined(__APPLE__) && defined(__MACH__) && defined(APPLE_HAS_MEMSET_S) memset_s(dst, len_, 0, len_); #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) || defined(__OpenBSD__) explicit_bzero(dst, len_); diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index d4a90220beafb5..f85982f33735d3 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -257,11 +257,11 @@ FStar_UInt128_gte_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) { FStar_UInt128_uint128 lit; lit.low = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); lit.high = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); return lit; } @@ -294,14 +294,12 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul32(uint64_t x, uint32_t y) { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), + FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * (uint64_t)y)); lit.high = - ((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) + ((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32; return lit; } @@ -315,28 +313,19 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul_wide(uint64_t x, uint64_t { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), + FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y))); lit.high = - (x >> FStar_UInt128_u32_32) - * (y >> FStar_UInt128_u32_32) - + - (((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) + (x >> FStar_UInt128_u32_32) * (y >> FStar_UInt128_u32_32) + + (((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32) + - ((FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) + ((FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) >> FStar_UInt128_u32_32); return lit; } diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index d91650b44bb4e7..a6776282423b95 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=7720f6d4fc0468a99d5ea6120976bcc271e42727 +expected_hacl_star_rev=4ef25b547b377dcef855db4289c6a00580e7221c hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" From dda5dc5f06871e871deb322318de7b4a50589baa Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 14:26:37 +0200 Subject: [PATCH 109/713] [3.14] gh-132388: test HACL* and OpenSSL hash functions in pure Python HMAC (GH-134051) (#134089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-132388: test HACL* and OpenSSL hash functions in pure Python HMAC (GH-134051) (cherry picked from commit 73d71a416fb05b64c2b43fade5d781a1fa0cb2cd) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/support/hashlib_helper.py | 221 ++++++++++++++++++++++++++++- Lib/test/test_hmac.py | 154 ++++++++++++-------- 2 files changed, 308 insertions(+), 67 deletions(-) diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index 5043f08dd93de6..7032257b06877a 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -23,6 +23,22 @@ def requires_builtin_hmac(): return unittest.skipIf(_hmac is None, "requires _hmac") +def _missing_hash(digestname, implementation=None, *, exc=None): + parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] + msg = " ".join(filter(None, parts)) + raise unittest.SkipTest(msg) from exc + + +def _openssl_availabillity(digestname, *, usedforsecurity): + try: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + except AttributeError: + assert _hashlib is None + _missing_hash(digestname, "OpenSSL") + except ValueError as exc: + _missing_hash(digestname, "OpenSSL", exc=exc) + + def _decorate_func_or_class(func_or_class, decorator_func): if not isinstance(func_or_class, type): return decorator_func(func_or_class) @@ -71,8 +87,7 @@ def wrapper(*args, **kwargs): try: test_availability() except ValueError as exc: - msg = f"missing hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) from exc + _missing_hash(digestname, exc=exc) return func(*args, **kwargs) return wrapper @@ -87,14 +102,44 @@ def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): The hashing algorithm may be missing or blocked by a strict crypto policy. """ def decorator_func(func): - @requires_hashlib() + @requires_hashlib() # avoid checking at each call @functools.wraps(func) def wrapper(*args, **kwargs): + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + return func(*args, **kwargs) + return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): + """Find the OpenSSL hash function constructor by its name.""" + assert isinstance(digestname, str), digestname + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return getattr(_hashlib, f"openssl_{digestname}") + + +def requires_builtin_hashdigest( + module_name, digestname, *, usedforsecurity=True +): + """Decorator raising SkipTest if a HACL* hashing algorithm is missing. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + module = import_module(module_name) try: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - except ValueError: - msg = f"missing OpenSSL hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) + getattr(module, digestname) + except AttributeError: + fullname = f'{module_name}.{digestname}' + _missing_hash(fullname, implementation="HACL") return func(*args, **kwargs) return wrapper @@ -103,6 +148,168 @@ def decorator(func_or_class): return decorator +def find_builtin_hashdigest_constructor( + module_name, digestname, *, usedforsecurity=True +): + """Find the HACL* hash function constructor. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + module = import_module(module_name) + try: + constructor = getattr(module, digestname) + constructor(b'', usedforsecurity=usedforsecurity) + except (AttributeError, TypeError, ValueError): + _missing_hash(f'{module_name}.{digestname}', implementation="HACL") + return constructor + + +class HashFunctionsTrait: + """Mixin trait class containing hash functions. + + This class is assumed to have all unitest.TestCase methods but should + not directly inherit from it to prevent the test suite being run on it. + + Subclasses should implement the hash functions by returning an object + that can be recognized as a valid digestmod parameter for both hashlib + and HMAC. In particular, it cannot be a lambda function as it will not + be recognized by hashlib (it will still be accepted by the pure Python + implementation of HMAC). + """ + + ALGORITHMS = [ + 'md5', 'sha1', + 'sha224', 'sha256', 'sha384', 'sha512', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + ] + + # Default 'usedforsecurity' to use when looking up a hash function. + usedforsecurity = True + + def _find_constructor(self, name): + # By default, a missing algorithm skips the test that uses it. + self.assertIn(name, self.ALGORITHMS) + self.skipTest(f"missing hash function: {name}") + + @property + def md5(self): + return self._find_constructor("md5") + + @property + def sha1(self): + return self._find_constructor("sha1") + + @property + def sha224(self): + return self._find_constructor("sha224") + + @property + def sha256(self): + return self._find_constructor("sha256") + + @property + def sha384(self): + return self._find_constructor("sha384") + + @property + def sha512(self): + return self._find_constructor("sha512") + + @property + def sha3_224(self): + return self._find_constructor("sha3_224") + + @property + def sha3_256(self): + return self._find_constructor("sha3_256") + + @property + def sha3_384(self): + return self._find_constructor("sha3_384") + + @property + def sha3_512(self): + return self._find_constructor("sha3_512") + + +class NamedHashFunctionsTrait(HashFunctionsTrait): + """Trait containing named hash functions. + + Hash functions are available if and only if they are available in hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return name + + +class OpenSSLHashFunctionsTrait(HashFunctionsTrait): + """Trait containing OpenSSL hash functions. + + Hash functions are available if and only if they are available in _hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return find_openssl_hashdigest_constructor( + name, usedforsecurity=self.usedforsecurity + ) + + +class BuiltinHashFunctionsTrait(HashFunctionsTrait): + """Trait containing HACL* hash functions. + + Hash functions are available if and only if they are available in C. + In particular, HACL* HMAC-MD5 may be available even though HACL* md5 + is not since the former is unconditionally built. + """ + + def _find_constructor_in(self, module, name): + self.assertIn(name, self.ALGORITHMS) + return find_builtin_hashdigest_constructor(module, name) + + @property + def md5(self): + return self._find_constructor_in("_md5", "md5") + + @property + def sha1(self): + return self._find_constructor_in("_sha1", "sha1") + + @property + def sha224(self): + return self._find_constructor_in("_sha2", "sha224") + + @property + def sha256(self): + return self._find_constructor_in("_sha2", "sha256") + + @property + def sha384(self): + return self._find_constructor_in("_sha2", "sha384") + + @property + def sha512(self): + return self._find_constructor_in("_sha2", "sha512") + + @property + def sha3_224(self): + return self._find_constructor_in("_sha3", "sha3_224") + + @property + def sha3_256(self): + return self._find_constructor_in("_sha3","sha3_256") + + @property + def sha3_384(self): + return self._find_constructor_in("_sha3","sha3_384") + + @property + def sha3_512(self): + return self._find_constructor_in("_sha3","sha3_512") + + def find_gil_minsize(modules_names, default=2048): """Get the largest GIL_MINSIZE value for the given cryptographic modules. diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 70c7943772249e..e898644dd8a552 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -1,8 +1,27 @@ +"""Test suite for HMAC. + +Python provides three different implementations of HMAC: + +- OpenSSL HMAC using OpenSSL hash functions. +- HACL* HMAC using HACL* hash functions. +- Generic Python HMAC using user-defined hash functions. + +The generic Python HMAC implementation is able to use OpenSSL +callables or names, HACL* named hash functions or arbitrary +objects implementing PEP 247 interface. + +In the two first cases, Python HMAC wraps a C HMAC object (either OpenSSL +or HACL*-based). As a last resort, HMAC is re-implemented in pure Python. +It is however interesting to test the pure Python implementation against +the OpenSSL and HACL* hash functions. +""" + import binascii import functools import hmac import hashlib import random +import test.support import test.support.hashlib_helper as hashlib_helper import types import unittest @@ -10,6 +29,12 @@ import warnings from _operator import _compare_digest as operator_compare_digest from test.support import check_disallow_instantiation +from test.support.hashlib_helper import ( + BuiltinHashFunctionsTrait, + HashFunctionsTrait, + NamedHashFunctionsTrait, + OpenSSLHashFunctionsTrait, +) from test.support.import_helper import import_fresh_module, import_module try: @@ -382,50 +407,7 @@ class BuiltinAssertersMixin(ThroughBuiltinAPIMixin, AssertersMixin): pass -class HashFunctionsTrait: - """Trait class for 'hashfunc' in hmac_new() and hmac_digest().""" - - ALGORITHMS = [ - 'md5', 'sha1', - 'sha224', 'sha256', 'sha384', 'sha512', - 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - ] - - # By default, a missing algorithm skips the test that uses it. - _ = property(lambda self: self.skipTest("missing hash function")) - md5 = sha1 = _ - sha224 = sha256 = sha384 = sha512 = _ - sha3_224 = sha3_256 = sha3_384 = sha3_512 = _ - del _ - - -class WithOpenSSLHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with an OpenSSL-based callable 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - @property - @hashlib_helper.requires_openssl_hashdigest(name) - def func(self, *, __name=name): # __name needed to bind 'name' - return getattr(_hashlib, f'openssl_{__name}') - setattr(cls, name, func) - - -class WithNamedHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with a named 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - setattr(cls, name, name) - - -class RFCTestCaseMixin(AssertersMixin, HashFunctionsTrait): +class RFCTestCaseMixin(HashFunctionsTrait, AssertersMixin): """Test HMAC implementations against RFC 2202/4231 and NIST test vectors. - Test vectors for MD5 and SHA-1 are taken from RFC 2202. @@ -739,26 +721,83 @@ def test_sha3_512_nist(self): ) -class PyRFCTestCase(ThroughObjectMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PurePythonInitHMAC(PyModuleMixin, HashFunctionsTrait): + + @classmethod + def setUpClass(cls): + super().setUpClass() + for meth in ['_init_openssl_hmac', '_init_builtin_hmac']: + fn = getattr(cls.hmac.HMAC, meth) + cm = mock.patch.object(cls.hmac.HMAC, meth, autospec=True, wraps=fn) + cls.enterClassContext(cm) + + @classmethod + def tearDownClass(cls): + cls.hmac.HMAC._init_openssl_hmac.assert_not_called() + cls.hmac.HMAC._init_builtin_hmac.assert_not_called() + # Do not assert that HMAC._init_old() has been called as it's tricky + # to determine whether a test for a specific hash function has been + # executed or not. On regular builds, it will be called but if a + # hash function is not available, it's hard to detect for which + # test we should checj HMAC._init_old() or not. + super().tearDownClass() + + +class PyRFCOpenSSLTestCase(ThroughObjectMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.HMAC(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). """ -class PyDotNewRFCTestCase(ThroughModuleAPIMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PyRFCBuiltinTestCase(ThroughObjectMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.HMAC(). + + The underlying hash functions are HACL*-based but + _init_old() is used instead of _init_builtin_hmac(). + """ + + +class PyDotNewOpenSSLRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.new(). + + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class PyDotNewBuiltinRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.new(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are HACL-based but + _init_old() is used instead of _init_openssl_hmac(). """ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """OpenSSL implementation of HMAC. @@ -767,7 +806,8 @@ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, class BuiltinRFCTestCase(BuiltinAssertersMixin, - WithNamedHashFunctions, RFCTestCaseMixin, + NamedHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """Built-in HACL* implementation of HMAC. @@ -784,12 +824,6 @@ def assert_hmac_extra_cases( self.check_hmac_hexdigest(key, msg, hexdigest, digest_size, func) -# TODO(picnixz): once we have a HACL* HMAC, we should also test the Python -# implementation of HMAC with a HACL*-based hash function. For now, we only -# test it partially via the '_sha2' module, but for completeness we could -# also test the RFC test vectors against all possible implementations. - - class DigestModTestCaseMixin(CreatorMixin, DigestMixin): """Tests for the 'digestmod' parameter for hmac_new() and hmac_digest().""" From fe5636ea96b25635a657b281f0148a0533ec1caf Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 14:29:40 +0200 Subject: [PATCH 110/713] [3.14] gh-124210: Add introduction to `threading` docs (GH-127046) (#134090) Co-authored-by: Semyon Moroz Co-authored-by: Peter Bierma --- Doc/library/threading.rst | 114 +++++++++++++++++++++++++++++--------- 1 file changed, 89 insertions(+), 25 deletions(-) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 249c0a5cb035c3..7edcdcabdce1df 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -11,6 +11,52 @@ This module constructs higher-level threading interfaces on top of the lower level :mod:`_thread` module. +.. include:: ../includes/wasm-notavail.rst + +Introduction +------------ + +The :mod:`!threading` module provides a way to run multiple `threads +`_ (smaller +units of a process) concurrently within a single process. It allows for the +creation and management of threads, making it possible to execute tasks in +parallel, sharing memory space. Threads are particularly useful when tasks are +I/O bound, such as file operations or making network requests, +where much of the time is spent waiting for external resources. + +A typical use case for :mod:`!threading` includes managing a pool of worker +threads that can process multiple tasks concurrently. Here's a basic example of +creating and starting threads using :class:`~threading.Thread`:: + + import threading + import time + + def crawl(link, delay=3): + print(f"crawl started for {link}") + time.sleep(delay) # Blocking I/O (simulating a network request) + print(f"crawl ended for {link}") + + links = [ + "https://python.org", + "https://docs.python.org", + "https://peps.python.org", + ] + + # Start threads for each link + threads = [] + for link in links: + # Using `args` to pass positional arguments and `kwargs` for keyword arguments + t = threading.Thread(target=crawl, args=(link,), kwargs={"delay": 2}) + threads.append(t) + + # Start each thread + for t in threads: + t.start() + + # Wait for all threads to finish + for t in threads: + t.join() + .. versionchanged:: 3.7 This module used to be optional, it is now always available. @@ -45,7 +91,25 @@ level :mod:`_thread` module. However, threading is still an appropriate model if you want to run multiple I/O-bound tasks simultaneously. -.. include:: ../includes/wasm-notavail.rst +GIL and performance considerations +---------------------------------- + +Unlike the :mod:`multiprocessing` module, which uses separate processes to +bypass the :term:`global interpreter lock` (GIL), the threading module operates +within a single process, meaning that all threads share the same memory space. +However, the GIL limits the performance gains of threading when it comes to +CPU-bound tasks, as only one thread can execute Python bytecode at a time. +Despite this, threads remain a useful tool for achieving concurrency in many +scenarios. + +As of Python 3.13, experimental :term:`free-threaded ` builds +can disable the GIL, enabling true parallel execution of threads, but this +feature is not available by default (see :pep:`703`). + +.. TODO: At some point this feature will become available by default. + +Reference +--------- This module defines the following functions: @@ -62,7 +126,7 @@ This module defines the following functions: Return the current :class:`Thread` object, corresponding to the caller's thread of control. If the caller's thread of control was not created through the - :mod:`threading` module, a dummy thread object with limited functionality is + :mod:`!threading` module, a dummy thread object with limited functionality is returned. The function ``currentThread`` is a deprecated alias for this function. @@ -157,13 +221,13 @@ This module defines the following functions: .. index:: single: trace function - Set a trace function for all threads started from the :mod:`threading` module. + Set a trace function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.settrace` for each thread, before its :meth:`~Thread.run` method is called. .. function:: settrace_all_threads(func) - Set a trace function for all threads started from the :mod:`threading` module + Set a trace function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.settrace` for each thread, before its @@ -186,13 +250,13 @@ This module defines the following functions: .. index:: single: profile function - Set a profile function for all threads started from the :mod:`threading` module. + Set a profile function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.setprofile` for each thread, before its :meth:`~Thread.run` method is called. .. function:: setprofile_all_threads(func) - Set a profile function for all threads started from the :mod:`threading` module + Set a profile function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.setprofile` for each thread, before its @@ -257,8 +321,8 @@ when implemented, are mapped to module-level functions. All of the methods described below are executed atomically. -Thread-Local Data ------------------ +Thread-local data +^^^^^^^^^^^^^^^^^ Thread-local data is data whose values are thread specific. If you have data that you want to be local to a thread, create a @@ -389,8 +453,8 @@ affects what we see:: .. _thread-objects: -Thread Objects --------------- +Thread objects +^^^^^^^^^^^^^^ The :class:`Thread` class represents an activity that is run in a separate thread of control. There are two ways to specify the activity: by passing a @@ -645,8 +709,8 @@ since it is impossible to detect the termination of alien threads. .. _lock-objects: -Lock Objects ------------- +Lock objects +^^^^^^^^^^^^ A primitive lock is a synchronization primitive that is not owned by a particular thread when locked. In Python, it is currently the lowest level @@ -738,8 +802,8 @@ All methods are executed atomically. .. _rlock-objects: -RLock Objects -------------- +RLock objects +^^^^^^^^^^^^^ A reentrant lock is a synchronization primitive that may be acquired multiple times by the same thread. Internally, it uses the concepts of "owning thread" @@ -848,8 +912,8 @@ call release as many times the lock has been acquired can lead to deadlock. .. _condition-objects: -Condition Objects ------------------ +Condition objects +^^^^^^^^^^^^^^^^^ A condition variable is always associated with some kind of lock; this can be passed in or one will be created by default. Passing one in is useful when @@ -1026,8 +1090,8 @@ item to the buffer only needs to wake up one consumer thread. .. _semaphore-objects: -Semaphore Objects ------------------ +Semaphore objects +^^^^^^^^^^^^^^^^^ This is one of the oldest synchronization primitives in the history of computer science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he @@ -1107,7 +1171,7 @@ Semaphores also support the :ref:`context management protocol `. .. _semaphore-examples: -:class:`Semaphore` Example +:class:`Semaphore` example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Semaphores are often used to guard resources with limited capacity, for example, @@ -1135,8 +1199,8 @@ causes the semaphore to be released more than it's acquired will go undetected. .. _event-objects: -Event Objects -------------- +Event objects +^^^^^^^^^^^^^ This is one of the simplest mechanisms for communication between threads: one thread signals an event and other threads wait for it. @@ -1192,8 +1256,8 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. _timer-objects: -Timer Objects -------------- +Timer objects +^^^^^^^^^^^^^ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` @@ -1230,8 +1294,8 @@ For example:: only work if the timer is still in its waiting stage. -Barrier Objects ---------------- +Barrier objects +^^^^^^^^^^^^^^^ .. versionadded:: 3.2 From 2d00484f30a7189bd0e2a4c9b3df7a10173ab72c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 17:37:01 +0200 Subject: [PATCH 111/713] [3.14] gh-133515: fix docs for unawaited coroutines in debug mode (GH-134081) (#134093) gh-133515: fix docs for unawaited coroutines in debug mode (GH-134081) (cherry picked from commit d94b1e9cac82143048031530e6c51e59f597bccd) Co-authored-by: Kumar Aditya --- Doc/library/asyncio-dev.rst | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 44b507a9811116..7831b613bd4a60 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -46,10 +46,6 @@ In addition to enabling the debug mode, consider also: When the debug mode is enabled: -* asyncio checks for :ref:`coroutines that were not awaited - ` and logs them; this mitigates - the "forgotten await" pitfall. - * Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and :meth:`loop.call_at` methods) raise an exception if they are called from a wrong thread. From c8dcde4d5e34893b2017dee6ee50db641514ef34 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 21:20:47 +0200 Subject: [PATCH 112/713] [3.14] gh-133286: add explanation about `seq` for pathlib Pattern Language (GH-133340) (#134105) gh-133286: add explanation about `seq` for pathlib Pattern Language (GH-133340) (cherry picked from commit ac8df4b5892d2e4bd99731e7d87223a35c238f81) Co-authored-by: alexey semenyuk --- Doc/library/pathlib.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 7d7692dea5c38c..86351e65dc4ed6 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1781,9 +1781,12 @@ The following wildcards are supported in patterns for ``?`` Matches one non-separator character. ``[seq]`` - Matches one character in *seq*. + Matches one character in *seq*, where *seq* is a sequence of characters. + Range expressions are supported; for example, ``[a-z]`` matches any lowercase ASCII letter. + Multiple ranges can be combined: ``[a-zA-Z0-9_]`` matches any ASCII letter, digit, or underscore. + ``[!seq]`` - Matches one character not in *seq*. + Matches one character not in *seq*, where *seq* follows the same rules as above. For a literal match, wrap the meta-characters in brackets. For example, ``"[?]"`` matches the character ``"?"``. From 7a5851dc902a86d76c194caede9c24e98ec70a37 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 21:23:52 +0200 Subject: [PATCH 113/713] [3.14] gh-117026: Remove outdated sentence in SimpleHTTPRequestHandler docs (GH-117027) (GH-134107) The code was changed in 0f7cddc308b297e6a1c2dd61503acea38401656f (bpo-839496/gh-39531). (cherry picked from commit ea2d707bd59963bd4f53407108026930ff12ae56) Co-authored-by: Saleh Dehqanpour --- Doc/library/http.server.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 54df4a7e804d50..02016c789b24b4 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -429,8 +429,7 @@ instantiation, of which this module provides three different variants: ``'Last-Modified:'`` header with the file's modification time. Then follows a blank line signifying the end of the headers, and then the - contents of the file are output. If the file's MIME type starts with - ``text/`` the file is opened in text mode; otherwise binary mode is used. + contents of the file are output. For example usage, see the implementation of the ``test`` function in :source:`Lib/http/server.py`. From f89323236f4669a5b66ed0e8b2f584cb0cb27232 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 16 May 2025 22:56:24 +0200 Subject: [PATCH 114/713] [3.14] gh-134109: Fix showing comments in pydoc output for argparse (GH-134110) (GH-134112) Comments immediately preceding the object's source code are used if the object has no docstring. Comments that do not describe the object should be separated from the following source code by an empty line. (cherry picked from commit 71cf4dd622832848cace358a7f8444243afd2e83) Co-authored-by: Serhiy Storchaka --- Lib/argparse.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Lib/argparse.py b/Lib/argparse.py index f688c38d0d1ae5..d1a6350c3fda6d 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -205,6 +205,7 @@ def _set_color(self, color): # =============================== # Section and indentation methods # =============================== + def _indent(self): self._current_indent += self._indent_increment self._level += 1 @@ -256,6 +257,7 @@ def _add_item(self, func, args): # ======================== # Message building methods # ======================== + def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) @@ -299,6 +301,7 @@ def add_arguments(self, actions): # ======================= # Help-formatting methods # ======================= + def format_help(self): help = self._root_section.format_help() if help: @@ -1467,6 +1470,7 @@ def __init__(self, # ==================== # Registration methods # ==================== + def register(self, registry_name, value, object): registry = self._registries.setdefault(registry_name, {}) registry[value] = object @@ -1477,6 +1481,7 @@ def _registry_get(self, registry_name, value, default=None): # ================================== # Namespace default accessor methods # ================================== + def set_defaults(self, **kwargs): self._defaults.update(kwargs) @@ -1496,6 +1501,7 @@ def get_default(self, dest): # ======================= # Adding argument actions # ======================= + def add_argument(self, *args, **kwargs): """ add_argument(dest, ..., name=value, ...) @@ -1921,6 +1927,7 @@ def identity(string): # ======================= # Pretty __repr__ methods # ======================= + def _get_kwargs(self): names = [ 'prog', @@ -1935,6 +1942,7 @@ def _get_kwargs(self): # ================================== # Optional/Positional adding methods # ================================== + def add_subparsers(self, **kwargs): if self._subparsers is not None: raise ValueError('cannot have multiple subparser arguments') @@ -1988,6 +1996,7 @@ def _get_positional_actions(self): # ===================================== # Command line argument parsing methods # ===================================== + def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: @@ -2582,6 +2591,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== # Value conversion methods # ======================== + def _get_values(self, action, arg_strings): # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: @@ -2681,6 +2691,7 @@ def _check_value(self, action, value): # ======================= # Help-formatting methods # ======================= + def format_usage(self): formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, @@ -2718,6 +2729,7 @@ def _get_formatter(self): # ===================== # Help-printing methods # ===================== + def print_usage(self, file=None): if file is None: file = _sys.stdout @@ -2739,6 +2751,7 @@ def _print_message(self, message, file=None): # =============== # Exiting methods # =============== + def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) From 5cdad8c90caa4efbd7846764a37d08402cd5812c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 09:27:38 +0200 Subject: [PATCH 115/713] [3.14] gh-133889: Improve tests for SimpleHTTPRequestHandler (GH-134102) (GH-134121) (cherry picked from commit fcaf009907fc39d604907315155c1f1de811dd88) Co-authored-by: Serhiy Storchaka --- Lib/test/test_httpservers.py | 140 ++++++++++++++++++++++++----------- 1 file changed, 98 insertions(+), 42 deletions(-) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 2cafa4e45a1313..4cb78c87ea7670 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -522,34 +522,111 @@ def close_conn(): reader.close() return body + def check_list_dir_dirname(self, dirname, quotedname=None): + fullpath = os.path.join(self.tempdir, dirname) + try: + os.mkdir(os.path.join(self.tempdir, dirname)) + except (OSError, UnicodeEncodeError): + self.skipTest(f'Can not create directory {dirname!a} ' + f'on current file system') + + if quotedname is None: + quotedname = urllib.parse.quote(dirname, errors='surrogatepass') + response = self.request(self.base_url + '/' + quotedname + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + displaypath = html.escape(f'{self.base_url}/{dirname}/', quote=False) + enc = sys.getfilesystemencoding() + prefix = f'listing for {displaypath}', body) + self.assertIn(prefix + b'h1>', body) + + def check_list_dir_filename(self, filename): + fullpath = os.path.join(self.tempdir, filename) + content = ascii(fullpath).encode() + (os_helper.TESTFN_UNDECODABLE or b'\xff') + try: + with open(fullpath, 'wb') as f: + f.write(content) + except OSError: + self.skipTest(f'Can not create file {filename!a} ' + f'on current file system') + + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + quotedname = urllib.parse.quote(filename, errors='surrogatepass') + enc = response.headers.get_content_charset() + self.assertIsNotNone(enc) + self.assertIn((f'href="{quotedname}"').encode('ascii'), body) + displayname = html.escape(filename, quote=False) + self.assertIn(f'>{displayname}<'.encode(enc, 'surrogateescape'), body) + + response = self.request(self.base_url + '/' + quotedname) + self.check_status_and_reason(response, HTTPStatus.OK, data=content) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_dirname(self): + dirname = os_helper.TESTFN_NONASCII + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_filename(self): + filename = os_helper.TESTFN_NONASCII + '.txt' + self.check_list_dir_filename(filename) + @unittest.skipIf(is_apple, 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'need os_helper.TESTFN_UNDECODABLE') - def test_undecodable_filename(self): - enc = sys.getfilesystemencoding() + def test_list_dir_undecodable_dirname(self): + dirname = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_filename(self): filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' - with open(os.path.join(self.tempdir, filename), 'wb') as f: - f.write(os_helper.TESTFN_UNDECODABLE) - response = self.request(self.base_url + '/') - if is_apple: - # On Apple platforms the HFS+ filesystem replaces bytes that - # aren't valid UTF-8 into a percent-encoded value. - for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). - filename = name - break - body = self.check_status_and_reason(response, HTTPStatus.OK) - quotedname = urllib.parse.quote(filename, errors='surrogatepass') - self.assertIn(('href="%s"' % quotedname) - .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename, quote=False)) - .encode(enc, 'surrogateescape'), body) - response = self.request(self.base_url + '/' + quotedname) - self.check_status_and_reason(response, HTTPStatus.OK, - data=os_helper.TESTFN_UNDECODABLE) + self.check_list_dir_filename(filename) + + def test_list_dir_undecodable_dirname2(self): + dirname = '\ufffd.dir' + self.check_list_dir_dirname(dirname, quotedname='%ff.dir') + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_dirname(self): + dirname = os_helper.TESTFN_UNENCODABLE + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_filename(self): + filename = os_helper.TESTFN_UNENCODABLE + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_escape_dirname(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&', '&', '', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + dirname = name + '.dir' + self.check_list_dir_dirname(dirname, + quotedname=urllib.parse.quote(dirname, safe='&<>\'"')) + + def test_list_dir_escape_filename(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&', '&', '', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + filename = name + '.txt' + self.check_list_dir_filename(filename) + os_helper.unlink(os.path.join(self.tempdir, filename)) def test_undecodable_parameter(self): # sanity check using a valid parameter @@ -731,27 +808,6 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - def test_html_escape_filename(self): - filename = '.txt' - fullpath = os.path.join(self.tempdir, filename) - - try: - open(fullpath, 'wb').close() - except OSError: - raise unittest.SkipTest('Can not create file %s on current file ' - 'system' % filename) - - try: - response = self.request(self.base_url + '/') - body = self.check_status_and_reason(response, HTTPStatus.OK) - enc = response.headers.get_content_charset() - finally: - os.unlink(fullpath) # avoid affecting test_undecodable_filename - - self.assertIsNotNone(enc) - html_text = '>%s<' % html.escape(filename, quote=False) - self.assertIn(html_text.encode(enc), body) - cgi_file1 = """\ #!%s From d7230e1110f9dd210596349e09c5824862666b5b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 09:37:43 +0200 Subject: [PATCH 116/713] [3.14] gh-134098: Fix handling %-encoded trailing slash in SimpleHTTPRequestHandler (GH-134099) (GH-134123) (cherry picked from commit 2f1ecb3bc474a5895dce090cca7b8afe7b560040) Co-authored-by: Serhiy Storchaka --- Lib/http/server.py | 8 ++++---- Lib/test/test_httpservers.py | 11 +++++++++++ .../2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst | 2 ++ 3 files changed, 17 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 64f766f9bc2c1b..8be1903743a9a2 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -750,7 +750,7 @@ def send_head(self): f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): + if not parts.path.endswith(('/', '%2f', '%2F')): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', @@ -890,14 +890,14 @@ def translate_path(self, path): """ # abandon query parameters - path = path.split('?',1)[0] - path = path.split('#',1)[0] + path = path.split('#', 1)[0] + path = path.split('?', 1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) + trailing_slash = path.endswith('/') path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 4cb78c87ea7670..557e698aa3481c 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -692,10 +692,19 @@ def test_get(self): # check for trailing "/" which should return 404. See Issue17324 response = self.request(self.base_url + '/test/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2f') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2F') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request(self.base_url + '/') self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2f') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2F') + self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.base_url) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), self.base_url + "/") self.assertEqual(response.getheader("Content-Length"), "0") response = self.request(self.base_url + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) @@ -801,6 +810,8 @@ def test_path_without_leading_slash(self): self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/") response = self.request(self.tempdir_name + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name + '?hi=1') diff --git a/Misc/NEWS.d/next/Library/2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst b/Misc/NEWS.d/next/Library/2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst new file mode 100644 index 00000000000000..32eff5371c4f1e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst @@ -0,0 +1,2 @@ +Fix handling paths that end with a percent-encoded slash (``%2f`` or +``%2F``) in :class:`http.server.SimpleHTTPRequestHandler`. From 075cabb02650e786008e45b632816c6a723801b0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 10:05:47 +0200 Subject: [PATCH 117/713] [3.14] gh-133881: add forward reference to `list.sort()` in lambda expression tutorial (GH-133910) (#134126) gh-133881: add forward reference to `list.sort()` in lambda expression tutorial (GH-133910) (cherry picked from commit c1c9ad1d5a62a591eb2f0f0d29f3fa02e0949f14) Co-authored-by: Oleg Burnaev <51371645+Shepard2154@users.noreply.github.com> --- Doc/tutorial/controlflow.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 95939242fb7d44..9a810f16388b9d 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -999,7 +999,8 @@ scope:: 43 The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: +is to pass a small function as an argument. For instance, :meth:`list.sort` +takes a sorting key function *key* which can be a lambda function:: >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] >>> pairs.sort(key=lambda pair: pair[1]) From 851b26f4bedb37591fc39e7f3d47dc078f34e7f9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 10:06:04 +0200 Subject: [PATCH 118/713] [3.14] gh-113878: fix `versionadded` in `dataclasses.field()` documentation (GH-134065) (#134128) gh-113878: fix `versionadded` in `dataclasses.field()` documentation (GH-134065) (cherry picked from commit 9d73875072e20bf3cc87cadc553d6a6fb4f71df5) Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> --- Doc/library/dataclasses.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 72612211b43d5e..f18c7cc9c02da6 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -304,9 +304,9 @@ Module contents .. versionadded:: 3.10 - - ``doc``: optional docstring for this field. + - *doc*: optional docstring for this field. - .. versionadded:: 3.13 + .. versionadded:: 3.14 If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be From 9f1307afa0e5af8f107a23203b78720a3cfbafc2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 11:04:42 +0200 Subject: [PATCH 119/713] [3.14] Docs: C API: Improve documentation around non-Python threads with subinterpreters (GH-131087) (GH-134130) Docs: C API: Improve documentation around non-Python threads with subinterpreters (GH-131087) (cherry picked from commit af6b3b825f3b653ffdb29fc1dd36de8acfe0a641) Co-authored-by: Peter Bierma Co-authored-by: Victor Stinner --- Doc/c-api/init.rst | 59 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index d538ef18fbc25c..4d0c39dc9a19d2 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1083,8 +1083,36 @@ Note that the ``PyGILState_*`` functions assume there is only one global interpreter (created automatically by :c:func:`Py_Initialize`). Python supports the creation of additional interpreters (using :c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. +``PyGILState_*`` API is unsupported. This is because :c:func:`PyGILState_Ensure` +and similar functions default to :term:`attaching ` a +:term:`thread state` for the main interpreter, meaning that the thread can't safely +interact with the calling subinterpreter. + +Supporting subinterpreters in non-Python threads +------------------------------------------------ + +If you would like to support subinterpreters with non-Python created threads, you +must use the ``PyThreadState_*`` API instead of the traditional ``PyGILState_*`` +API. + +In particular, you must store the interpreter state from the calling +function and pass it to :c:func:`PyThreadState_New`, which will ensure that +the :term:`thread state` is targeting the correct interpreter:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = ThreadData->interp; + PyThreadState *tstate = PyThreadState_New(interp); + PyThreadState_Swap(tstate); + + /* GIL of the subinterpreter is now held. + Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); .. _fork-and-threads: @@ -1261,6 +1289,10 @@ code, or when embedding the Python interpreter: .. seealso: :c:func:`PyEval_ReleaseThread` + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + The following functions use thread-local storage, and are not compatible with sub-interpreters: @@ -1287,10 +1319,10 @@ with sub-interpreters: When the function returns, there will be an :term:`attached thread state` and the thread will be able to call arbitrary Python code. Failure is a fatal error. - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to :ref:`cautions-regarding-runtime-finalization` for more details. .. versionchanged:: 3.14 @@ -1307,7 +1339,6 @@ with sub-interpreters: Every call to :c:func:`PyGILState_Ensure` must be matched by a call to :c:func:`PyGILState_Release` on the same thread. - .. c:function:: PyThreadState* PyGILState_GetThisThreadState() Get the :term:`attached thread state` for this thread. May return ``NULL`` if no @@ -1315,20 +1346,30 @@ with sub-interpreters: always has such a thread-state, even if no auto-thread-state call has been made on the main thread. This is mainly a helper/diagnostic function. - .. seealso: :c:func:`PyThreadState_Get`` + .. note:: + This function does not account for :term:`thread states ` created + by something other than :c:func:`PyGILState_Ensure` (such as :c:func:`PyThreadState_New`). + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. seealso: :c:func:`PyThreadState_Get`` .. c:function:: int PyGILState_Check() Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the :term:`GIL` will it return ``1``. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. This is mainly a helper/diagnostic function. It can be useful for example in callback contexts or memory allocation functions when knowing that the :term:`GIL` is locked can allow the caller to perform sensitive actions or otherwise behave differently. + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + .. versionadded:: 3.4 From b43e1d0b48584af16fd0c4bda4c0d56bf72bf57a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 14:03:33 +0200 Subject: [PATCH 120/713] [3.14] Docs: fix spelling of "test case" in `unittest` documentation (GH-134137) (#134138) Docs: fix spelling of "test case" in `unittest` documentation (GH-134137) (cherry picked from commit b41d79c776af4bfa5d28fc2036c9137978294b6a) Co-authored-by: Clifford Gama <53076065+cliff688@users.noreply.github.com> --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 61022fe052ca80..dcdda1719bf593 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -109,7 +109,7 @@ Here is a short script to test three string methods:: unittest.main() -A testcase is created by subclassing :class:`unittest.TestCase`. The three +A test case is created by subclassing :class:`unittest.TestCase`. The three individual tests are defined with methods whose names start with the letters ``test``. This naming convention informs the test runner about which methods represent tests. From cca18fcbe817a24bfeeb7f15a00158f8214e010d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 15:17:23 +0200 Subject: [PATCH 121/713] [3.14] gh-88275: Add missing `__init__` method to `match` example (GH-120281) (#134142) gh-88275: Add missing `__init__` method to `match` example (GH-120281) (cherry picked from commit 7a9d46295a497669eaa6e647c33ab71c8cf620a1) Co-authored-by: Kirill Podoprigora --- Doc/whatsnew/3.10.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 3c815721a92f8c..f8df802768a945 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -551,11 +551,12 @@ Patterns and classes If you are using classes to structure your data, you can use as a pattern the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: +pattern has the ability to capture instance attributes into variables:: class Point: - x: int - y: int + def __init__(self, x, y): + self.x = x + self.y = y def location(point): match point: From 1ba5e65b76eb0d745e63691bda7206e66f72a961 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 17 May 2025 21:48:54 +0200 Subject: [PATCH 122/713] [3.14] gh-134119: Fix crash from calling next() on exhausted template iterator (GH-134120) (#134153) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-134119: Fix crash from calling next() on exhausted template iterator (GH-134120) (cherry picked from commit fc7f4c36664314393bd4c30355e21bd7aeac524d) Co-authored-by: Jelle Zijlstra Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_string/test_templatelib.py | 7 +++++++ .../2025-05-16-20-59-12.gh-issue-134119.w8expI.rst | 2 ++ Objects/templateobject.c | 3 +++ 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-20-59-12.gh-issue-134119.w8expI.rst diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index 5b9490c2be6de0..85fcff486d6616 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -148,6 +148,13 @@ def test_iter(self): self.assertEqual(res[1].format_spec, '') self.assertEqual(res[2], ' yz') + def test_exhausted(self): + # See https://github.com/python/cpython/issues/134119. + template_iter = iter(t"{1}") + self.assertIsInstance(next(template_iter), Interpolation) + self.assertRaises(StopIteration, next, template_iter) + self.assertRaises(StopIteration, next, template_iter) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-20-59-12.gh-issue-134119.w8expI.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-20-59-12.gh-issue-134119.w8expI.rst new file mode 100644 index 00000000000000..754e816628563e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-20-59-12.gh-issue-134119.w8expI.rst @@ -0,0 +1,2 @@ +Fix crash when calling :func:`next` on an exhausted template string iterator. +Patch by Jelle Zijlstra. diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 06cb19e0b6d056..4293a311c440f7 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -23,6 +23,9 @@ templateiter_next(PyObject *op) if (self->from_strings) { item = PyIter_Next(self->stringsiter); self->from_strings = 0; + if (item == NULL) { + return NULL; + } if (PyUnicode_GET_LENGTH(item) == 0) { Py_SETREF(item, PyIter_Next(self->interpolationsiter)); self->from_strings = 1; From eaee2ae0333dde4f18bf849e4a90c8eebb7e2cb9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 00:50:00 +0200 Subject: [PATCH 123/713] [3.14] gh-134064: Fix sys.remote_exec() error checking (GH-134067) (#134162) gh-134064: Fix sys.remote_exec() error checking (GH-134067) (cherry picked from commit 009e7b36981fd07f7cca1fdcfcf172ce1584fac7) Co-authored-by: Victor Stinner --- Lib/test/test_sys.py | 7 +++++++ Python/sysmodule.c | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 8af2e3488b48d9..fb1c8492a64d38 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -2176,6 +2176,13 @@ def test_remote_exec_invalid_pid(self): with self.assertRaises(OSError): sys.remote_exec(99999, "print('should not run')") + def test_remote_exec_invalid_script(self): + """Test remote exec with invalid script type""" + with self.assertRaises(TypeError): + sys.remote_exec(0, None) + with self.assertRaises(TypeError): + sys.remote_exec(0, 123) + def test_remote_exec_syntax_error(self): """Test remote exec with syntax error in script""" script = ''' diff --git a/Python/sysmodule.c b/Python/sysmodule.c index b46c2ab7c4483d..8692caa238a64d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2484,7 +2484,7 @@ sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) PyObject *path; const char *debugger_script_path; - if (PyUnicode_FSConverter(script, &path) < 0) { + if (PyUnicode_FSConverter(script, &path) == 0) { return NULL; } debugger_script_path = PyBytes_AS_STRING(path); From bf39decabdc7feff3a7441fa93136c22596c2d67 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 03:54:00 +0200 Subject: [PATCH 124/713] [3.14] gh-134150: Clarify distinction between JSON and Python objects (GH-134154) (#134166) gh-134150: Clarify distinction between JSON and Python objects (GH-134154) * gh-134150: Clarify distinction between JSON objects and Python objects in json module docs * Revert change to JSON introduction * Clarify occurrences of "object literal" as JSON (cherry picked from commit fa4e088668d4a41f9be5babe7edd5409290ee92a) Co-authored-by: Micha Albert --- Doc/library/json.rst | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 26579ec6328860..12a5a96a3c56f3 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -18,12 +18,17 @@ is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). +.. note:: + The term "object" in the context of JSON processing in Python can be + ambiguous. All values in Python are objects. In JSON, an object refers to + any data wrapped in curly braces, similar to a Python dictionary. + .. warning:: Be cautious when parsing JSON data from untrusted sources. A malicious JSON string may cause the decoder to consume considerable CPU and memory resources. Limiting the size of data to be parsed is recommended. -:mod:`json` exposes an API familiar to users of the standard library +This module exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. Encoding basic Python object hierarchies:: @@ -60,7 +65,7 @@ Pretty printing:: "6": 7 } -Specializing JSON object encoding:: +Customizing JSON object encoding:: >>> import json >>> def custom_json(obj): @@ -83,7 +88,7 @@ Decoding JSON:: >>> json.load(io) ['streaming API'] -Specializing JSON object decoding:: +Customizing JSON object decoding:: >>> import json >>> def as_complex(dct): @@ -279,7 +284,7 @@ Basic Usage :param object_hook: If set, a function that is called with the result of - any object literal decoded (a :class:`dict`). + any JSON object literal decoded (a :class:`dict`). The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders, @@ -289,7 +294,7 @@ Basic Usage :param object_pairs_hook: If set, a function that is called with the result of - any object literal decoded with an ordered list of pairs. + any JSON object literal decoded with an ordered list of pairs. The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders. From 8d51ed6b050b2678a8c1ad22b739554171a563cc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 17:08:40 +0200 Subject: [PATCH 125/713] [3.14] gh-134100: Fix use-after-free in `PyImport_ImportModuleLevelObject` (GH-134117) (#134171) gh-134100: Fix use-after-free in `PyImport_ImportModuleLevelObject` (GH-134117) (cherry picked from commit 4e9005d32ff466925f40af410f2ea6bf2329bcf8) Co-authored-by: Nico-Posada <102486290+Nico-Posada@users.noreply.github.com> --- .../import_/test_relative_imports.py | 15 +++++++++++++++ ...2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst | 2 ++ Python/import.c | 4 +++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index e535d119763148..1549cbe96ce2d1 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -223,6 +223,21 @@ def test_relative_import_no_package_exists_absolute(self): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst new file mode 100644 index 00000000000000..d672347f9ad246 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst @@ -0,0 +1,2 @@ +Fix a use-after-free bug that occurs when an imported module isn't +in :data:`sys.modules` after its initial import. Patch by Nico-Posada. diff --git a/Python/import.c b/Python/import.c index afdc28eda31b9b..f24c5716feb6c2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3852,15 +3852,17 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } final_mod = import_get_module(tstate, to_return); - Py_DECREF(to_return); if (final_mod == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_KeyError, "%R not in sys.modules as expected", to_return); } + Py_DECREF(to_return); goto error; } + + Py_DECREF(to_return); } } else { From bb5f92adcf3c35fe9842e564f0ff6d9e5f900ebe Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 17:29:19 +0200 Subject: [PATCH 126/713] [3.14] gh-134144: Fix use-after-free in zapthreads() (GH-134145) (#134182) gh-134144: Fix use-after-free in zapthreads() (GH-134145) (cherry picked from commit f2de1e6861c27bd498f598efc01600450979b5f9) Co-authored-by: b-pass --- Lib/test/test_interpreters/test_api.py | 8 +++++ ...-05-17-14-41-21.gh-issue-134144.xVpZik.rst | 1 + Modules/_testinternalcapi.c | 29 ++++++++++++++++--- Python/pystate.c | 9 ++++-- 4 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-05-17-14-41-21.gh-issue-134144.xVpZik.rst diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 66c7afce88f0d8..1e2d572b1cbb81 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1452,6 +1452,14 @@ def test_destroy(self): self.assertFalse( self.interp_exists(interpid)) + with self.subTest('basic C-API'): + interpid = _testinternalcapi.create_interpreter() + self.assertTrue( + self.interp_exists(interpid)) + _testinternalcapi.destroy_interpreter(interpid, basic=True) + self.assertFalse( + self.interp_exists(interpid)) + def test_get_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. diff --git a/Misc/NEWS.d/next/C_API/2025-05-17-14-41-21.gh-issue-134144.xVpZik.rst b/Misc/NEWS.d/next/C_API/2025-05-17-14-41-21.gh-issue-134144.xVpZik.rst new file mode 100644 index 00000000000000..11c7bd59a4dd1d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-17-14-41-21.gh-issue-134144.xVpZik.rst @@ -0,0 +1 @@ +Fix crash when calling :c:func:`Py_EndInterpreter` with a :term:`thread state` that isn't the initial thread for the interpreter. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 92f744c5a5fc70..76bd76cc6b2490 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1690,11 +1690,12 @@ create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) static PyObject * destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "basic", NULL}; PyObject *idobj = NULL; + int basic = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O:destroy_interpreter", kwlist, - &idobj)) + "O|p:destroy_interpreter", kwlist, + &idobj, &basic)) { return NULL; } @@ -1704,7 +1705,27 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - _PyXI_EndInterpreter(interp, NULL, NULL); + if (basic) + { + // Test the basic Py_EndInterpreter with weird out of order thread states + PyThreadState *t1, *t2; + PyThreadState *prev; + t1 = interp->threads.head; + if (t1 == NULL) { + t1 = PyThreadState_New(interp); + } + t2 = PyThreadState_New(interp); + prev = PyThreadState_Swap(t2); + PyThreadState_Clear(t1); + PyThreadState_Delete(t1); + Py_EndInterpreter(t2); + PyThreadState_Swap(prev); + } + else + { + // use the cross interpreter _PyXI_EndInterpreter normally + _PyXI_EndInterpreter(interp, NULL, NULL); + } Py_RETURN_NONE; } diff --git a/Python/pystate.c b/Python/pystate.c index 1ac134400856d4..14ae2748b0bc99 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1908,9 +1908,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) static void zapthreads(PyInterpreterState *interp) { + PyThreadState *tstate; /* No need to lock the mutex here because this should only happen - when the threads are all really dead (XXX famous last words). */ - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + when the threads are all really dead (XXX famous last words). + + Cannot use _Py_FOR_EACH_TSTATE_UNLOCKED because we are freeing + the thread states here. + */ + while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); From 11650c354fc71cd69777bafc44636b9ee6e7391f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 18:29:03 +0200 Subject: [PATCH 127/713] [3.14] fix indent in `controlflow.rst` docs (GH-134008) (#134184) fix indent in `controlflow.rst` docs (GH-134008) (cherry picked from commit 4ce91871a9077fcf8806d6fd79bdb32af9587306) Co-authored-by: Yongzi Li <1538321957@qq.com> --- Doc/tutorial/controlflow.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 9a810f16388b9d..5c0e8f34bf82f4 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -1056,7 +1056,7 @@ Here is an example of a multi-line docstring:: >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything. .. _tut-annotations: From 76957eb8ccd98b4647bfc655d145aadd05c977e0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 18:34:19 +0200 Subject: [PATCH 128/713] [3.14] document `Py_VISIT` as a macro in the docs (GH-133688) (#134186) document `Py_VISIT` as a macro in the docs (GH-133688) (cherry picked from commit bb32f3c698f5192dddd2d2f33c7c0a3d05afc223) Co-authored-by: da-woods --- Doc/c-api/gcsupport.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index d1f0982b818931..3e23605778f05a 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -180,9 +180,9 @@ provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` must name its arguments exactly *visit* and *arg*: -.. c:function:: void Py_VISIT(PyObject *o) +.. c:macro:: Py_VISIT(o) - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: From 70735878ab3fa8180ea21e6e1cd85092673c2a62 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 18 May 2025 21:16:37 +0200 Subject: [PATCH 129/713] [3.14] gh-133889: Only show the path of the URL in the SimpleHTTPRequestHandler page (GH-134135) (GH-134190) The query and fragment are ambiguous and not used. (cherry picked from commit 5cbc8c632e860941602e8f7da9aab52fae40aca6) Co-authored-by: Serhiy Storchaka --- Lib/http/server.py | 7 +++++-- Lib/test/test_httpservers.py | 13 +++++++------ .../2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst | 3 +++ 3 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 8be1903743a9a2..31fe9cae781f0a 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -840,11 +840,14 @@ def list_directory(self, path): return None list.sort(key=lambda a: a.lower()) r = [] + displaypath = self.path + displaypath = displaypath.split('#', 1)[0] + displaypath = displaypath.split('?', 1)[0] try: - displaypath = urllib.parse.unquote(self.path, + displaypath = urllib.parse.unquote(displaypath, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(self.path) + displaypath = urllib.parse.unquote(displaypath) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = f'Directory listing for {displaypath}' diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 557e698aa3481c..9a321952a8d61e 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -628,13 +628,14 @@ def test_list_dir_escape_filename(self): self.check_list_dir_filename(filename) os_helper.unlink(os.path.join(self.tempdir, filename)) - def test_undecodable_parameter(self): - # sanity check using a valid parameter + def test_list_dir_with_query_and_fragment(self): + prefix = f'listing for {self.base_url}/', response) + self.assertIn(prefix + b'h1>', response) response = self.request(self.base_url + '/?x=123').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=123'.encode('latin1')) - # now the bogus encoding - response = self.request(self.base_url + '/?x=%bb').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. diff --git a/Misc/NEWS.d/next/Library/2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst b/Misc/NEWS.d/next/Library/2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst new file mode 100644 index 00000000000000..58b213e29f98d8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst @@ -0,0 +1,3 @@ +The generated directory listing page in +:class:`http.server.SimpleHTTPRequestHandler` now only shows the decoded +path component of the requested URL, and not the query and fragment. From 91d71ac1bd68d44a8648e39e24b2f1c26575b8ba Mon Sep 17 00:00:00 2001 From: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Date: Sun, 18 May 2025 20:33:14 +0100 Subject: [PATCH 130/713] [3.14] gh-134060: Don't create a certain symlink in venv if platform does not support it (GH-134061) --- Lib/venv/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 15e15b7a5184c2..17ee28e826d5cf 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -315,7 +315,7 @@ def setup_python(self, context): os.chmod(path, 0o755) suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] - if sys.version_info[:2] == (3, 14): + if sys.version_info[:2] == (3, 14) and sys.getfilesystemencoding() == 'utf-8': suffixes.append('𝜋thon') for suffix in suffixes: path = os.path.join(binpath, suffix) From 9136ccfffef99db5743dd692d10b5d12e405a275 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 11:59:58 +0200 Subject: [PATCH 131/713] [3.14] gh-76023: Make os.path.realpath to ignore WinError 1005 in non-strict mode (GH-128328) (GH-134203) (cherry picked from commit d55e11b8049e3abf3cc187b4958224b225a39897) Co-authored-by: BecoKo --- Lib/ntpath.py | 3 ++- .../next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 5481bb8888ef59..52ff2af743af6c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -661,9 +661,10 @@ def _getfinalpathname_nonstrict(path): # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME # 161: ERROR_BAD_PATHNAME + # 1005: ERROR_UNRECOGNIZED_VOLUME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. diff --git a/Misc/NEWS.d/next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst b/Misc/NEWS.d/next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst new file mode 100644 index 00000000000000..958f4f4a4408f7 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst @@ -0,0 +1 @@ +Make :func:`os.path.realpath` ignore Windows error 1005 when in non-strict mode. From d9c08c57e93802f417fd11ee868cf98805c0bc6b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 13:26:27 +0200 Subject: [PATCH 132/713] [3.14] gh-133999: Fix `except` parsing regression in 3.14 (GH-134035) (#134206) gh-133999: Fix `except` parsing regression in 3.14 (GH-134035) (cherry picked from commit 84914ad0e5f96f0ca7238f3b4bc7fc4e50b1abb3) Co-authored-by: sobolevn --- Grammar/python.gram | 4 +-- Lib/test/test_syntax.py | 17 ++++++++++ ...-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst | 2 ++ Parser/parser.c | 32 +++++++++++++------ 4 files changed, 43 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 528075b8bc0d71..8b15cf29fa02e2 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1418,7 +1418,7 @@ invalid_except_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | 'except' expression 'as' a=expression { + | 'except' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) } invalid_except_star_stmt: @@ -1426,7 +1426,7 @@ invalid_except_star_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") } - | 'except' '*' expression 'as' a=expression { + | 'except' '*' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) } invalid_finally_stmt: diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 0ee17849e28121..fadafd9b3d3aea 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -1431,6 +1431,23 @@ Traceback (most recent call last): SyntaxError: cannot use except* statement with literal +Regression tests for gh-133999: + + >>> try: pass + ... except TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + + >>> try: pass + ... except* TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + + >>> match 1: + ... case 1 | 2 as abc: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 Traceback (most recent call last): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst new file mode 100644 index 00000000000000..7d9c49688c3ce2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst @@ -0,0 +1,2 @@ +Fix :exc:`SyntaxError` regression in :keyword:`except` parsing after +:gh:`123440`. diff --git a/Parser/parser.c b/Parser/parser.c index 509fac7df6e371..dc55e391d29280 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -24202,7 +24202,7 @@ invalid_try_stmt_rule(Parser *p) // | 'except' expression ',' expressions 'as' NAME ':' // | 'except' expression ['as' NAME] NEWLINE // | 'except' NEWLINE -// | 'except' expression 'as' expression +// | 'except' expression 'as' expression ':' block static void * invalid_except_stmt_rule(Parser *p) { @@ -24318,15 +24318,17 @@ invalid_except_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' NEWLINE")); } - { // 'except' expression 'as' expression + { // 'except' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; + Token * _literal; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' @@ -24336,9 +24338,13 @@ invalid_except_stmt_rule(Parser *p) (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except statement with %s" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24349,7 +24355,7 @@ invalid_except_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression ':' block")); } _res = NULL; done: @@ -24361,7 +24367,7 @@ invalid_except_stmt_rule(Parser *p) // | 'except' '*' expression ',' expressions 'as' NAME ':' // | 'except' '*' expression ['as' NAME] NEWLINE // | 'except' '*' (NEWLINE | ':') -// | 'except' '*' expression 'as' expression +// | 'except' '*' expression 'as' expression ':' block static void * invalid_except_star_stmt_rule(Parser *p) { @@ -24486,16 +24492,18 @@ invalid_except_star_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' (NEWLINE | ':')")); } - { // 'except' '*' expression 'as' expression + { // 'except' '*' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; Token * _literal; + Token * _literal_1; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' @@ -24507,9 +24515,13 @@ invalid_except_star_stmt_rule(Parser *p) (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except* statement with %s" , _PyPegen_get_expr_name ( a ) ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -24520,7 +24532,7 @@ invalid_except_star_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); } _res = NULL; done: From e3cc4008c4d73d100fcc0c69f57f925c7470d8d7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 15:49:46 +0200 Subject: [PATCH 133/713] [3.14] gh-122055: Clarify documentation for empty matches in RE (GH-133169) (GH-134217) (cherry picked from commit 44b73d3cd4466e148460883acf4494124eae8c91) Co-authored-by: Serhiy Storchaka --- Doc/library/re.rst | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Doc/library/re.rst b/Doc/library/re.rst index eb3b1e5549cc98..75ebbf11c8e47c 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -991,8 +991,8 @@ Functions That way, separator components are always found at the same relative indices within the result list. - Empty matches for the pattern split the string only when not adjacent - to a previous empty match. + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. .. code:: pycon @@ -1095,9 +1095,12 @@ Functions The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all - occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous empty match, so ``sub('x*', '-', 'abxd')`` returns - ``'-a-b--d-'``. + occurrences will be replaced. + + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. + As a result, ``sub('x*', '-', 'abxd')`` returns ``'-a-b--d-'`` + instead of ``'-a-b-d-'``. .. index:: single: \g; in regular expressions @@ -1128,8 +1131,7 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - Empty matches for the pattern are replaced when adjacent to a previous - non-empty match. + An empty match can occur immediately after a non-empty match. .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. From 182464eb9ed1a6d045cd84d4c1851d3804ebc6bd Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 16:13:44 +0200 Subject: [PATCH 134/713] [3.14] gh-134097: Print number of refs & blocks after each statement in new REPL (gh-134136) (gh-134220) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit c31547a5914db93b8b38c6a5261ef716255f3582) Co-authored-by: Kirill Podoprigora Co-authored-by: Łukasz Langa --- Lib/_pyrepl/simple_interact.py | 9 +++++++++ Lib/test/test_pyrepl/test_pyrepl.py | 15 ++++++++++++++- ...2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index b3848833e14208..c23894a34b8b53 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -110,6 +110,10 @@ def run_multiline_interactive_console( more_lines = functools.partial(_more_lines, console) input_n = 0 + _is_x_showrefcount_set = sys._xoptions.get("showrefcount") + _is_pydebug_build = hasattr(sys, "gettotalrefcount") + show_ref_count = _is_x_showrefcount_set and _is_pydebug_build + def maybe_run_command(statement: str) -> bool: statement = statement.strip() if statement in console.locals or statement not in REPL_COMMANDS: @@ -167,3 +171,8 @@ def maybe_run_command(statement: str) -> bool: except: console.showtraceback() console.resetbuffer() + if show_ref_count: + console.write( + f"[{sys.gettotalrefcount()} refs," + f" {sys.getallocatedblocks()} blocks]\n" + ) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index fc8114891d12dd..f80755b3368f8f 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -10,7 +10,7 @@ import tempfile from unittest import TestCase, skipUnless, skipIf from unittest.mock import patch -from test.support import force_not_colorized, make_clean_env +from test.support import force_not_colorized, make_clean_env, Py_DEBUG from test.support import SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -1610,3 +1610,16 @@ def test_prompt_after_help(self): # Extra stuff (newline and `exit` rewrites) are necessary # because of how run_repl works. self.assertNotIn(">>> \n>>> >>>", cleaned_output) + + @skipUnless(Py_DEBUG, '-X showrefcount requires a Python debug build') + def test_showrefcount(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + env["PYTHON_BASIC_REPL"] = "1" + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) diff --git a/Misc/NEWS.d/next/Library/2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst b/Misc/NEWS.d/next/Library/2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst new file mode 100644 index 00000000000000..0b388d9db380f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst @@ -0,0 +1 @@ +Fix interaction of the new :term:`REPL` and :option:`-X showrefcount <-X>` command line option. From a3c3d6f0efd6d1708ed648b461097373ac09005d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 16:39:05 +0200 Subject: [PATCH 135/713] [3.14] gh-134158: Fix PyREPL coloring of double braces in f/t-strings (gh-134159) (#134227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Loïc Simon Co-authored-by: Loïc Simon Co-authored-by: Peter Bierma Co-authored-by: Łukasz Langa --- Lib/_pyrepl/utils.py | 8 ++++- Lib/test/test_pyrepl/test_reader.py | 31 +++++++++++++++++++ ...-05-17-20-44-51.gh-issue-134158.ewLNLp.rst | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-44-51.gh-issue-134158.ewLNLp.rst diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 752049ac05acdf..e04fbdc6c8a5c4 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -41,9 +41,15 @@ def from_re(cls, m: Match[str], group: int | str) -> Self: @classmethod def from_token(cls, token: TI, line_len: list[int]) -> Self: + end_offset = -1 + if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} + and token.string.endswith(("{", "}"))): + # gh-134158: a visible trailing brace comes from a double brace in input + end_offset += 1 + return cls( line_len[token.start[0] - 1] + token.start[1], - line_len[token.end[0] - 1] + token.end[1] - 1, + line_len[token.end[0] - 1] + token.end[1] + end_offset, ) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 57526f88f9384b..1f655264f1c00a 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -517,6 +517,37 @@ def unfinished_function(): self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected) + def test_syntax_highlighting_literal_brace_in_fstring_or_tstring(self): + code = dedent( + """\ + f"{{" + f"}}" + f"a{{b" + f"a}}b" + f"a{{b}}c" + t"a{{b}}c" + f"{{{0}}}" + f"{ {0} }" + """ + ) + expected = dedent( + """\ + {s}f"{z}{s}<<{z}{s}"{z} + {s}f"{z}{s}>>{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a>>{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}t"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}f"{z}{s}<<{z}{o}<{z}{n}0{z}{o}>{z}{s}>>{z}{s}"{z} + {s}f"{z}{o}<{z} {o}<{z}{n}0{z}{o}>{z} {o}>{z}{s}"{z} + """ + ).format(**colors).replace("<", "{").replace(">", "}") + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.maxDiff=None + self.assert_screen_equal(reader, expected) + def test_control_characters(self): code = 'flag = "🏳️‍🌈"' events = code_to_events(code) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-44-51.gh-issue-134158.ewLNLp.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-44-51.gh-issue-134158.ewLNLp.rst new file mode 100644 index 00000000000000..7b8bab739c37c8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-44-51.gh-issue-134158.ewLNLp.rst @@ -0,0 +1 @@ +Fix coloring of double braces in f-strings and t-strings in the :term:`REPL`. From 7d995ea9fd45258efa8d51f70b92e0cacec74f38 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 16:51:46 +0200 Subject: [PATCH 136/713] [3.14] gh-134214: Fix test case in pyrepl (gh-134223) (gh-134229) (cherry picked from commit faebf87b3716f7103ee5410456972db36f4b3ada) Co-authored-by: Jessica Temporal --- Lib/test/test_pyrepl/test_pyrepl.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index f80755b3368f8f..a1cfbe4c3cadb7 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1055,11 +1055,15 @@ def test_parse(self): self.assertEqual(actual, parsed) # The parser should not get tripped up by any # other preceding statements - code = f'import xyz\n{code}' - with self.subTest(code=code): + _code = f'import xyz\n{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) - code = f'import xyz;{code}' - with self.subTest(code=code): + _code = f'import xyz;{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) def test_parse_error(self): From c869898f39a64113689b4d674b996d678dec2843 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 17:09:15 +0200 Subject: [PATCH 137/713] [3.14] gh-128045: Mark unknown opcodes as deopting to themselves (GH-128044) (#134228) * gh-128045: Mark unknown opcodes as deopting to themselves (GH-128044) --- Include/internal/pycore_opcode_metadata.h | 29 +++++++++++++++++++ .../opcode_metadata_generator.py | 7 +++++ 2 files changed, 36 insertions(+) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0e34074f1600a7..8d92e29c435056 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1787,6 +1787,35 @@ const uint8_t _PyOpcode_Caches[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, + [122] = 122, + [123] = 123, + [124] = 124, + [125] = 125, + [126] = 126, + [127] = 127, + [212] = 212, + [213] = 213, + [214] = 214, + [215] = 215, + [216] = 216, + [217] = 217, + [218] = 218, + [219] = 219, + [220] = 220, + [221] = 221, + [222] = 222, + [223] = 223, + [224] = 224, + [225] = 225, + [226] = 226, + [227] = 227, + [228] = 228, + [229] = 229, + [230] = 230, + [231] = 231, + [232] = 232, + [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 620e4b6f1f4a69..10567204dcc599 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -157,6 +157,13 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: if inst.family is not None: deopt = inst.family.name deopts.append((inst.name, deopt)) + defined = set(analysis.opmap.values()) + for i in range(256): + if i not in defined: + deopts.append((f'{i}', f'{i}')) + + assert len(deopts) == 256 + assert len(set(x[0] for x in deopts)) == 256 for name, deopt in sorted(deopts): out.emit(f"[{name}] = {deopt},\n") out.emit("};\n\n") From 89b744a772978ccd883c289c1a7066c1fb14ac04 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 17:22:14 +0200 Subject: [PATCH 138/713] [3.14] gh-125225: Fix column misalignment in help('topics') output (gh-125226) (gh-134225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 'help("topics")' output was misaligned due to "ASSIGNMENTEXPRESSIONS" exceeding the implicit maximum default column width of 19 characters. Reduced the number of columns from 4 to 3 in the listtopics() function to allow more space for longer topic names. (cherry picked from commit b22460c44d1bc597c96d4a3d27ad8373d7952820) Co-authored-by: Étienne Pelletier Co-authored-by: Łukasz Langa --- Lib/pydoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/pydoc.py b/Lib/pydoc.py index def76d076a2989..7528178fdcae97 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -2110,7 +2110,7 @@ def intro(self): self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -2142,7 +2142,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: From 7686c752b308358bccd5e76e8516872e844a1b70 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Mon, 19 May 2025 21:24:53 +0100 Subject: [PATCH 139/713] [3.14] gh-133779: Revert Windows generation of pyconfig.h and go back to a static header. (GH-133966) Extension builders must specify Py_GIL_DISABLED if they want to link to the free-threaded builds. This was usually the case already, but this change guarantees it in all circumstances. --- Doc/howto/free-threading-extensions.rst | 8 +++++ Doc/whatsnew/3.14.rst | 6 ++++ Include/Python.h | 13 ++++++++ Lib/sysconfig/__init__.py | 2 +- Lib/test/test_sysconfig.py | 5 +-- ...-05-13-13-25-27.gh-issue-133779.-YcTBz.rst | 6 ++++ PC/{pyconfig.h.in => pyconfig.h} | 22 +++++++++---- PCbuild/_freeze_module.vcxproj | 27 +--------------- PCbuild/pyproject.props | 8 ++--- PCbuild/pythoncore.vcxproj | 31 ++----------------- PCbuild/regen.targets | 14 ++++----- Tools/msi/dev/dev_files.wxs | 2 +- Tools/peg_generator/pegen/build.py | 2 ++ 13 files changed, 68 insertions(+), 78 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst rename PC/{pyconfig.h.in => pyconfig.h} (97%) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3f6ee517050bd8..5a3970f15d52d6 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build:: /* code that only runs in the free-threaded build */ #endif +.. note:: + + On Windows, this macro is not defined automatically, but must be specified + to the compiler when building. The :func:`sysconfig.get_config_var` function + can be used to determine whether the current running interpreter had the + macro defined. + + Module Initialization ===================== diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 11361289874c9d..8f39b99e38e3a5 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -829,6 +829,12 @@ Kumar Aditya, Edgar Margffoy, and many others. Some of these contributors are employed by Meta, which has continued to provide significant engineering resources to support this project. +From 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + .. _whatsnew314-pyrepl-highlighting: diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a3..f34d581f0b4c91 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -59,6 +59,14 @@ # include // __readgsqword() #endif +// Suppress known warnings in Python header files. +#if defined(_MSC_VER) +// Warning that alignas behaviour has changed. Doesn't affect us, because we +// never relied on the old behaviour. +#pragma warning(push) +#pragma warning(disable: 5274) +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" @@ -138,4 +146,9 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" +// Restore warning filter +#ifdef _MSC_VER +#pragma warning(pop) +#endif + #endif /* !Py_PYTHON_H */ diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index dad715eb087387..f93b98dd681536 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -468,7 +468,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 53e55383bf9c72..963cf753ce6178 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -531,13 +531,10 @@ def test_srcdir(self): Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) # /PC/pyconfig.h.in always exists even if unused - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) if os.name == 'nt': - # /pyconfig.h exists on Windows in a build tree - pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) diff --git a/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst b/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst new file mode 100644 index 00000000000000..550600d5eebf11 --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst @@ -0,0 +1,6 @@ +Reverts the change to generate different :file:`pyconfig.h` files based on +compiler settings, as it was frequently causing extension builds to break. +In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always +be defined explicitly when compiling for the experimental free-threaded +runtime. The :func:`sysconfig.get_config_var` function can be used to +determine whether the current runtime was compiled with that flag or not. diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h similarity index 97% rename from PC/pyconfig.h.in rename to PC/pyconfig.h index 9e70303868e5de..66fe2ab0998493 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h @@ -94,12 +94,22 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ -/* Define to 1 if you want to disable the GIL */ -/* Uncomment the definition for free-threaded builds, or define it manually - * when compiling extension modules. Note that we test with #ifdef, so - * defining as 0 will still disable the GIL. */ -#ifndef Py_GIL_DISABLED -/* #define Py_GIL_DISABLED 1 */ +/* _DEBUG implies Py_DEBUG */ +#ifdef _DEBUG +# define Py_DEBUG 1 +#endif + +/* Define to 1 when compiling for experimental free-threaded builds */ +#ifdef Py_GIL_DISABLED +/* We undefine if it was set to zero because all later checks are #ifdef. + * Note that non-Windows builds do not do this, and so every effort should + * be made to avoid defining the variable at all when not desired. However, + * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely + * that a build backend will define it with the value. + */ +#if Py_GIL_DISABLED == 0 +#undef Py_GIL_DISABLED +#endif #endif /* Compiler specific defines */ diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8c161835af9a8c..efff6a58d895cb 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -276,7 +276,7 @@ - + @@ -436,31 +436,6 @@ - - - - - - - - - - @(PyConfigH->'%(FullPath)', ';') - $([System.IO.File]::ReadAllText($(PyConfigH))) - $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - - - $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) - - - - - diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 4e414dc913b9d5..7272542e13a5ca 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,10 +10,9 @@ $(MSBuildThisFileDirectory)obj\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\ $(IntDir.Replace(`\\`, `\`)) - - $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\ $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\ + $(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration) $(ProjectName) $(TargetName)$(PyDebugExt) false @@ -49,11 +48,12 @@ <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64; <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition) <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)"; + <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1; - $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories) - WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions) + $(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories) + WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)%(PreprocessorDefinitions) _Py_USING_PGO=1;%(PreprocessorDefinitions) MaxSpeed diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 549d6284972afc..32a8f2dbad3d5e 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -102,6 +102,7 @@ /Zm200 %(AdditionalOptions) $(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories) $(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories) + $(GeneratedJitStencilsDir);%(AdditionalIncludeDirectories) _USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions) _Py_HAVE_ZLIB;%(PreprocessorDefinitions) _Py_JIT;%(PreprocessorDefinitions) @@ -409,7 +410,7 @@ - + @@ -688,34 +689,6 @@ - - - - - - - - - @(PyConfigH->'%(FullPath)', ';') - $([System.IO.File]::ReadAllText($(PyConfigH))) - $([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h')) - - - $(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1')) - - - - - - - - - - - git diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 3ad17737807235..21de614e71ddce 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -29,12 +29,12 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> - <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils.h"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -116,7 +116,7 @@ @@ -126,7 +126,7 @@ $(JITArgs) --debug + WorkingDirectory="$(GeneratedJitStencilsDir)"/> diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 4357dc86d9d356..21f9c848cc6be5 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ - + diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 41338c29bdd9eb..be289c352de585 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -108,6 +108,8 @@ def compile_c_extension( extra_compile_args.append("-DPy_BUILD_CORE_MODULE") # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c extra_compile_args.append("-D_Py_TEST_PEGEN") + if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"): + extra_compile_args.append("-DPy_GIL_DISABLED") extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST") if keep_asserts: extra_compile_args.append("-UNDEBUG") From 7cbc3ea8fb1b0f1ac082466e4fc99b740357e675 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 19 May 2025 23:46:50 +0200 Subject: [PATCH 140/713] [3.14] gh-134235: Import Autocomplete for Builtin Modules (GH-134277) (#134285) gh-134235: Import Autocomplete for Builtin Modules (GH-134277) * added enhancement auto completing import with sys builtins --------- (cherry picked from commit 8421b03b16a4852a527256cb7cdce2ab2d318548) Co-authored-by: Tom Wang <85062819+tommix626@users.noreply.github.com> Co-authored-by: Hunter --- Lib/_pyrepl/_module_completer.py | 5 +++-- Lib/test/test_pyrepl/test_pyrepl.py | 20 +++++++++++++++++++ ...-05-19-15-05-24.gh-issue-134235.pz9PwV.rst | 2 ++ 3 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-19-15-05-24.gh-issue-134235.pz9PwV.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 347f05607c75c5..0606797226d1e0 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -81,8 +81,9 @@ def find_modules(self, path: str, prefix: str) -> list[str]: def _find_modules(self, path: str, prefix: str) -> list[str]: if not path: # Top-level import (e.g. `import foo`` or `from foo`)` - return [name for _, name, _ in self.global_cache - if name.startswith(prefix)] + builtin_modules = [name for name in sys.builtin_module_names if name.startswith(prefix)] + third_party_modules = [name for _, name, _ in self.global_cache if name.startswith(prefix)] + return sorted(builtin_modules + third_party_modules) if path.startswith('.'): # Convert relative path to absolute path diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index a1cfbe4c3cadb7..59f5d1f893f9fd 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -959,6 +959,26 @@ def test_import_completions(self): output = reader.readline() self.assertEqual(output, expected) + def test_builtin_completion_top_level(self): + import importlib + # Make iter_modules() search only the standard library. + # This makes the test more reliable in case there are + # other user packages/scripts on PYTHONPATH which can + # intefere with the completions. + lib_path = os.path.dirname(importlib.__path__[0]) + sys.path = [lib_path] + + cases = ( + ("import bui\t\n", "import builtins"), + ("from bui\t\n", "from builtins"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + def test_relative_import_completions(self): cases = ( ("from .readl\t\n", "from .readline"), diff --git a/Misc/NEWS.d/next/Library/2025-05-19-15-05-24.gh-issue-134235.pz9PwV.rst b/Misc/NEWS.d/next/Library/2025-05-19-15-05-24.gh-issue-134235.pz9PwV.rst new file mode 100644 index 00000000000000..a65df886919145 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-15-05-24.gh-issue-134235.pz9PwV.rst @@ -0,0 +1,2 @@ +Updated tab completion on REPL to include builtin modules. Contributed by +Tom Wang, Hunter Young From e20f05f53fe6a54435923105329c9de24a10fd2e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 02:05:26 +0200 Subject: [PATCH 141/713] [3.14] gh-134201: Expand explanation of Base85 encodings in base64 docs (GH-134288) (#134298) gh-134201: Expand explanation of Base85 encodings in base64 docs (GH-134288) Explain history of de-facto standard and how to pick between the two Base-85 encoding functions in the base-64 module. --------- (cherry picked from commit 66aaad61037785639aec393be7618cb54b1372dc) Co-authored-by: Alek Binion Co-authored-by: C.A.M. Gerlach --- Doc/library/base64.rst | 48 ++++++++++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 834ab2536e6c14..529a7242443820 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -15,14 +15,9 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`4648`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`4648` encodings are suitable for encoding binary data so that it can be -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. +This includes the :ref:`encodings specified in ` +:rfc:`4648` (Base64, Base32 and Base16) +and the non-standard :ref:`Base85 encodings `. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -30,7 +25,7 @@ supports encoding :term:`bytes-like objects ` to ASCII strings containing ASCII to :class:`bytes`. Both base-64 alphabets defined in :rfc:`4648` (normal, and URL- and filesystem-safe) are supported. -The legacy interface does not support decoding from strings, but it does +The :ref:`legacy interface ` does not support decoding from strings, but it does provide functions for encoding and decoding to and from :term:`file objects `. It only supports the Base64 standard alphabet, and it adds newlines every 76 characters as per :rfc:`2045`. Note that if you are looking @@ -46,7 +41,15 @@ package instead. Any :term:`bytes-like objects ` are now accepted by all encoding and decoding functions in this module. Ascii85/Base85 support added. -The modern interface provides: + +.. _base64-rfc-4648: + +RFC 4648 Encodings +------------------ + +The :rfc:`4648` encodings are suitable for encoding binary data so that it can be +safely sent by email, used as parts of URLs, or included as part of an HTTP +POST request. .. function:: b64encode(s, altchars=None) @@ -181,6 +184,26 @@ The modern interface provides: incorrectly padded or if there are non-alphabet characters present in the input. +.. _base64-base-85: + +Base85 Encodings +----------------- + +Base85 encoding is not formally specified but rather a de facto standard, +thus different systems perform the encoding differently. + +The :func:`a85encode` and :func:`b85encode` functions in this module are two implementations of +the de facto standard. You should call the function with the Base85 +implementation used by the software you intend to work with. + +The two functions present in this module differ in how they handle the following: + +* Whether to include enclosing ``<~`` and ``~>`` markers +* Whether to include newline characters +* The set of ASCII characters used for encoding +* Handling of null bytes + +Refer to the documentation of the individual functions for more information. .. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) @@ -262,7 +285,10 @@ The modern interface provides: .. versionadded:: 3.13 -The legacy interface: +.. _base64-legacy: + +Legacy Interface +---------------- .. function:: decode(input, output) From 4effd060fcb309560c0fcbb2201954752cacf72e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 04:25:24 +0200 Subject: [PATCH 142/713] [3.14] gh-132983: Convert zstd ``__new__`` methods to Argument Clinic (GH-133860) (#133915) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/data/python3.14.abi | 2441 +++++++++-------- .../pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - .../internal/pycore_runtime_init_generated.h | 1 - .../internal/pycore_unicodeobject_generated.h | 4 - Lib/test/test_zstd.py | 24 +- Modules/_zstd/_zstdmodule.c | 65 +- Modules/_zstd/_zstdmodule.h | 5 +- Modules/_zstd/buffer.h | 5 +- Modules/_zstd/clinic/_zstdmodule.c.h | 22 +- Modules/_zstd/clinic/compressor.c.h | 28 +- Modules/_zstd/clinic/decompressor.c.h | 22 +- Modules/_zstd/clinic/zstddict.c.h | 48 +- Modules/_zstd/compressor.c | 148 +- Modules/_zstd/decompressor.c | 119 +- Modules/_zstd/zstddict.c | 144 +- Modules/_zstd/zstddict.h | 9 +- 17 files changed, 1534 insertions(+), 1553 deletions(-) diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index 88792d211ae2c3..c9959bef98fd07 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -1141,12 +1141,18 @@ + + + + + + @@ -1228,6 +1234,7 @@ + @@ -1283,6 +1290,10 @@ + + + + @@ -1783,7 +1794,7 @@ - + @@ -3970,72 +3981,72 @@ - - + + - + - - - - + + + + - + - - + + - + - - + + - - + + - + - - + + - + - + - - + + - - - - + + + + - - - + + + - - + + - - + + @@ -6475,16 +6486,40 @@ - - + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + - - + + @@ -7163,101 +7198,101 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - - - - + + + + + + @@ -7269,11 +7304,6 @@ - - - - - @@ -8081,89 +8111,94 @@ - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + + + + + + - - + + @@ -8803,116 +8838,136 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - - - + + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - - + + + - - - + + + - + - - - + + + - - + + - - - - + + + + - - + + - - + + @@ -14391,10 +14446,10 @@ - + - + @@ -17293,33 +17348,33 @@ - - + + - + - - + + - + - + - + - + - + - + - + @@ -17932,18 +17987,18 @@ - + - + - - + + - - + + @@ -18113,7 +18168,7 @@ - + @@ -19132,1246 +19187,1243 @@ - + - + - - + + - - + + - + - + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - - + + - + - - + + - + - + - + - - + + - + - - + + - - + + - - + + - + - + - + - + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - + - + - - + + - + - + - + - + - - + + - + - - + + - + - + - + - - + + - + - + - - + + - + - - + + - + - + - - + + - + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - + - + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - + - + - + - + - - + + - + - + - - + + - + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - + - + - - + + - - + + - + - - + + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - + - + - - + + - + - + - + - + - - + + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - - - - + - - + + - - + + @@ -20502,20 +20554,20 @@ - + - + - + - + - + - + @@ -20909,7 +20961,7 @@ - + @@ -21119,7 +21171,7 @@ - + @@ -21127,7 +21179,7 @@ - + @@ -21283,7 +21335,7 @@ - + @@ -21971,12 +22023,12 @@ - + - + @@ -21989,19 +22041,19 @@ - + - + - + - + - + @@ -22013,7 +22065,7 @@ - + @@ -22143,7 +22195,7 @@ - + @@ -23620,6 +23672,11 @@ + + + + + @@ -24429,7 +24486,7 @@ - + @@ -24887,7 +24944,7 @@ - + @@ -26766,33 +26823,33 @@ - + - + - + - + - + - + - + - + - + - - + + @@ -26803,65 +26860,65 @@ - - + + - + - + - + - - - - + + + + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + @@ -26885,6 +26942,14 @@ + + + + + + + + @@ -26892,6 +26957,10 @@ + + + + @@ -26946,6 +27015,10 @@ + + + + @@ -26988,147 +27061,159 @@ - + - - + + - - - - - - + + + + + + - - - - - - + + + + + + - - - + + + - - - + + + - - - - + + + + - - + + - - - - + + + + - - + + - - - - + + + + - - + + + + + + + + + + + + + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + - - - - + + + + - - - - + + + + - - + + - - + + - - - - + + + + - - + + - - - - - + + + + + - - - - + + + + @@ -28228,24 +28313,24 @@ - + - + - + - + - + - + - + @@ -28547,116 +28632,116 @@ - - - - + + + + - - - - - + + + + + - - + + - + - + - - + + - - - + + + - - - + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - + + + - - - - + + + + - - - - + + + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - - + + + - + - - - + + + @@ -29637,19 +29722,19 @@ - + - + - + - + @@ -30573,100 +30658,92 @@ - - - - - - - - - - - + + + - - - - + + + + - - - - + + + + - - - - - - + + + + + + - - - - - - - + + + + + + + - - - - - - - + + + + + + + - - - + + + - - - - + + + + - - - - - + + + + + - - + + - - - - + + + + - - - - - + + + + + - - - + + + - - - + + + diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9bde4faaf5a040..14fe9cdf0c77a0 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -904,7 +904,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3a83fd6b6042e2..197362a3e98be0 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -395,7 +395,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) STRUCT_FOR_ID(dict) - STRUCT_FOR_ID(dict_content) STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(digest) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4a34ffa559e124..cb6e9c0b02ce2c 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -902,7 +902,6 @@ extern "C" { INIT_ID(deterministic), \ INIT_ID(device), \ INIT_ID(dict), \ - INIT_ID(dict_content), \ INIT_ID(dictcomp), \ INIT_ID(difference_update), \ INIT_ID(digest), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index fefacef77c89ee..dbb0f4e921149c 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1368,10 +1368,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dict_content); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 83d922d9cf22ac..53ca592ea38828 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -288,8 +288,8 @@ def test_unknown_compression_parameter(self): KEY = 100001234 option = {CompressionParameter.compression_level: 10, KEY: 200000000} - pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY + pattern = (r'Invalid zstd compression parameter.*?' + fr'"unknown parameter \(key {KEY}\)"') with self.assertRaisesRegex(ZstdError, pattern): ZstdCompressor(options=option) @@ -420,8 +420,8 @@ def test_unknown_decompression_parameter(self): KEY = 100001234 options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], KEY: 200000000} - pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY + pattern = (r'Invalid zstd decompression parameter.*?' + fr'"unknown parameter \(key {KEY}\)"') with self.assertRaisesRegex(ZstdError, pattern): ZstdDecompressor(options=options) @@ -507,7 +507,7 @@ def test_decompress_epilogue_flags(self): self.assertFalse(d.needs_input) def test_decompressor_arg(self): - zd = ZstdDict(b'12345678', True) + zd = ZstdDict(b'12345678', is_raw=True) with self.assertRaises(TypeError): d = ZstdDecompressor(zstd_dict={}) @@ -1021,6 +1021,10 @@ def test_decompressor_skippable(self): class ZstdDictTestCase(unittest.TestCase): def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + # content < 8 b = b'1234567' with self.assertRaises(ValueError): @@ -1068,9 +1072,9 @@ def test_invalid_dict(self): # corrupted zd = ZstdDict(dict_content, is_raw=False) - with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): ZstdCompressor(zstd_dict=zd.as_digested_dict) - with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): ZstdDecompressor(zd) # wrong type @@ -1096,7 +1100,7 @@ def test_train_dict(self): TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) - ZstdDict(TRAINED_DICT.dict_content, False) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) self.assertNotEqual(TRAINED_DICT.dict_id, 0) self.assertGreater(len(TRAINED_DICT.dict_content), 0) @@ -1250,7 +1254,7 @@ def _nbytes(dat): def test_as_prefix(self): # V1 V1 = THIS_FILE_BYTES - zd = ZstdDict(V1, True) + zd = ZstdDict(V1, is_raw=True) # V2 mid = len(V1) // 2 @@ -1266,7 +1270,7 @@ def test_as_prefix(self): self.assertEqual(decompress(dat, zd.as_prefix), V2) # use wrong prefix - zd2 = ZstdDict(SAMPLES[0], True) + zd2 = ZstdDict(SAMPLES[0], is_raw=True) try: decompressed = decompress(dat, zd2.as_prefix) except ZstdError: # expected diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index c3852fe89732bc..0294828aa106ea 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -34,17 +31,17 @@ set_zstd_error(const _zstd_state* const state, switch (type) { case ERR_DECOMPRESS: - msg = "Unable to decompress zstd data: %s"; + msg = "Unable to decompress Zstandard data: %s"; break; case ERR_COMPRESS: - msg = "Unable to compress zstd data: %s"; + msg = "Unable to compress Zstandard data: %s"; break; case ERR_LOAD_D_DICT: - msg = "Unable to load zstd dictionary or prefix for decompression: %s"; + msg = "Unable to load Zstandard dictionary or prefix for decompression: %s"; break; case ERR_LOAD_C_DICT: - msg = "Unable to load zstd dictionary or prefix for compression: %s"; + msg = "Unable to load Zstandard dictionary or prefix for compression: %s"; break; case ERR_GET_C_BOUNDS: @@ -58,10 +55,10 @@ set_zstd_error(const _zstd_state* const state, break; case ERR_TRAIN_DICT: - msg = "Unable to train zstd dictionary: %s"; + msg = "Unable to train the Zstandard dictionary: %s"; break; case ERR_FINALIZE_DICT: - msg = "Unable to finalize zstd dictionary: %s"; + msg = "Unable to finalize the Zstandard dictionary: %s"; break; default: @@ -152,7 +149,7 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } if (ZSTD_isError(bounds.error)) { PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid.", + "Invalid zstd %s parameter \"%s\".", type, name); return; } @@ -187,13 +184,13 @@ _zstd.train_dict The size of the dictionary. / -Internal function, train a zstd dictionary on sample data. +Train a Zstandard dictionary on sample data. [clinic start generated code]*/ static PyObject * _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/ +/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/ { // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict // are pretty similar. We should see if we can refactor them to share that code. @@ -258,7 +255,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, chunk_sizes, (uint32_t)chunks_number); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { _zstd_state* const mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); @@ -292,10 +289,10 @@ _zstd.finalize_dict dict_size: Py_ssize_t The size of the dictionary. compression_level: int - Optimize for a specific zstd compression level, 0 means default. + Optimize for a specific Zstandard compression level, 0 means default. / -Internal function, finalize a zstd dictionary. +Finalize a Zstandard dictionary. [clinic start generated code]*/ static PyObject * @@ -303,7 +300,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size, int compression_level) -/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/ +/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; @@ -360,7 +357,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /* Parameters */ - /* Optimize for a specific zstd compression level, 0 means default. */ + /* Optimize for a specific Zstandard compression level, 0 means default. */ params.compressionLevel = compression_level; /* Write log to stderr, 0 = none. */ params.notificationLevel = 0; @@ -376,7 +373,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, (uint32_t)chunks_number, params); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { _zstd_state* const mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); @@ -407,12 +404,12 @@ _zstd.get_param_bounds is_compress: bool True for CompressionParameter, False for DecompressionParameter. -Internal function, get CompressionParameter/DecompressionParameter bounds. +Get CompressionParameter/DecompressionParameter bounds. [clinic start generated code]*/ static PyObject * _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) -/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/ +/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/ { ZSTD_bounds bound; if (is_compress) { @@ -442,14 +439,12 @@ _zstd.get_frame_size A bytes-like object, it should start from the beginning of a frame, and contains at least one complete frame. -Get the size of a zstd frame, including frame header and 4-byte checksum if it has one. - -It will iterate all blocks' headers within a frame, to accumulate the frame size. +Get the size of a Zstandard frame, including the header and optional checksum. [clinic start generated code]*/ static PyObject * _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/ +/*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/ { size_t frame_size; @@ -457,9 +452,9 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) if (ZSTD_isError(frame_size)) { _zstd_state* const mod_state = get_zstd_state(module); PyErr_Format(mod_state->ZstdError, - "Error when finding the compressed size of a zstd frame. " - "Make sure the frame_buffer argument starts from the " - "beginning of a frame, and its length not less than this " + "Error when finding the compressed size of a Zstandard frame. " + "Ensure the frame_buffer argument starts from the " + "beginning of a frame, and its length is not less than this " "complete frame. Zstd error message: %s.", ZSTD_getErrorName(frame_size)); return NULL; @@ -472,14 +467,14 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) _zstd.get_frame_info frame_buffer: Py_buffer - A bytes-like object, containing the header of a zstd frame. + A bytes-like object, containing the header of a Zstandard frame. -Internal function, get zstd frame infomation from a frame header. +Get Zstandard frame infomation from a frame header. [clinic start generated code]*/ static PyObject * _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/ +/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/ { uint64_t decompressed_size; uint32_t dict_id; @@ -494,9 +489,9 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) _zstd_state* const mod_state = get_zstd_state(module); PyErr_SetString(mod_state->ZstdError, "Error when getting information from the header of " - "a zstd frame. Make sure the frame_buffer argument " + "a Zstandard frame. Ensure the frame_buffer argument " "starts from the beginning of a frame, and its length " - "not less than the frame header (6~18 bytes)."); + "is not less than the frame header (6~18 bytes)."); return NULL; } @@ -518,13 +513,13 @@ _zstd.set_parameter_types d_parameter_type: object(subclass_of='&PyType_Type') DecompressionParameter IntEnum type object -Internal function, set CompressionParameter/DecompressionParameter types for validity check. +Set CompressionParameter and DecompressionParameter types for validity check. [clinic start generated code]*/ static PyObject * _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, PyObject *d_parameter_type) -/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/ +/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/ { _zstd_state* const mod_state = get_zstd_state(module); diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 00e0d2177f31f6..b36486442c6567 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index c902eef4f8e037..bff3a81d8aa11c 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index fc9f49813df6f1..766e1cfa776767 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -13,7 +13,7 @@ PyDoc_STRVAR(_zstd_train_dict__doc__, "train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" -"Internal function, train a zstd dictionary on sample data.\n" +"Train a Zstandard dictionary on sample data.\n" "\n" " samples_bytes\n" " Concatenation of samples.\n" @@ -73,7 +73,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__, " dict_size, compression_level, /)\n" "--\n" "\n" -"Internal function, finalize a zstd dictionary.\n" +"Finalize a Zstandard dictionary.\n" "\n" " custom_dict_bytes\n" " Custom dictionary content.\n" @@ -84,7 +84,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__, " dict_size\n" " The size of the dictionary.\n" " compression_level\n" -" Optimize for a specific zstd compression level, 0 means default."); +" Optimize for a specific Zstandard compression level, 0 means default."); #define _ZSTD_FINALIZE_DICT_METHODDEF \ {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, @@ -149,7 +149,7 @@ PyDoc_STRVAR(_zstd_get_param_bounds__doc__, "get_param_bounds($module, /, parameter, is_compress)\n" "--\n" "\n" -"Internal function, get CompressionParameter/DecompressionParameter bounds.\n" +"Get CompressionParameter/DecompressionParameter bounds.\n" "\n" " parameter\n" " The parameter to get bounds.\n" @@ -220,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__, "get_frame_size($module, /, frame_buffer)\n" "--\n" "\n" -"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n" +"Get the size of a Zstandard frame, including the header and optional checksum.\n" "\n" " frame_buffer\n" " A bytes-like object, it should start from the beginning of a frame,\n" -" and contains at least one complete frame.\n" -"\n" -"It will iterate all blocks\' headers within a frame, to accumulate the frame size."); +" and contains at least one complete frame."); #define _ZSTD_GET_FRAME_SIZE_METHODDEF \ {"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__}, @@ -291,10 +289,10 @@ PyDoc_STRVAR(_zstd_get_frame_info__doc__, "get_frame_info($module, /, frame_buffer)\n" "--\n" "\n" -"Internal function, get zstd frame infomation from a frame header.\n" +"Get Zstandard frame infomation from a frame header.\n" "\n" " frame_buffer\n" -" A bytes-like object, containing the header of a zstd frame."); +" A bytes-like object, containing the header of a Zstandard frame."); #define _ZSTD_GET_FRAME_INFO_METHODDEF \ {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, @@ -359,7 +357,7 @@ PyDoc_STRVAR(_zstd_set_parameter_types__doc__, "set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "--\n" "\n" -"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" +"Set CompressionParameter and DecompressionParameter types for validity check.\n" "\n" " c_parameter_type\n" " CompressionParameter IntEnum type object\n" @@ -428,4 +426,4 @@ _zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=8445b658dcdcbb9c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index d7909cdf89fcd1..f69161b590e5b7 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -8,30 +8,30 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, "ZstdCompressor(level=None, options=None, zstd_dict=None)\n" "--\n" "\n" "Create a compressor object for compressing data incrementally.\n" "\n" " level\n" -" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n" +" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n" " options\n" " A dict object that contains advanced compression parameters.\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" "Thread-safe at method level. For one-shot compression, use the compress()\n" "function instead."); -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict); +static PyObject * +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict); -static int -_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 3 @@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) } zstd_dict = fastargs[2]; skip_optional_pos: - return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict); + return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict); exit: return return_value; @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since zstd data consists\n" -"of one or more independent frames, the compressor object can still be used\n" -"after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard data\n" +"consists of one or more independent frames, the compressor object can still\n" +"be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -252,4 +252,4 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee2d1dc298de790c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 9359c637203f8f..4ecb19e9bde6ed 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -10,28 +10,28 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, "ZstdDecompressor(zstd_dict=None, options=None)\n" "--\n" "\n" "Create a decompressor object for decompressing data incrementally.\n" "\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" "Thread-safe at method level. For one-shot decompression, use the decompress()\n" "function instead."); -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options); +static PyObject * +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options); -static int -_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 2 @@ -82,7 +82,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs } options = fastargs[1]; skip_optional_pos: - return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options); + return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options); exit: return return_value; @@ -130,7 +130,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, "Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n" "\n" " data\n" -" A bytes-like object, zstd data to be decompressed.\n" +" A bytes-like object, Zstandard data to be decompressed.\n" " max_length\n" " Maximum size of returned data. When it is negative, the size of\n" " output buffer is unlimited. When it is nonnegative, returns at\n" @@ -227,4 +227,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 4e0f7b64172a74..34e0e4b3ecfe72 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -9,35 +9,33 @@ preserve #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, -"ZstdDict(dict_content, is_raw=False)\n" +PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, +"ZstdDict(dict_content, /, *, is_raw=False)\n" "--\n" "\n" -"Represents a zstd dictionary, which can be used for compression/decompression.\n" +"Represents a Zstandard dictionary.\n" "\n" " dict_content\n" -" A bytes-like object, dictionary\'s content.\n" +" The content of a Zstandard dictionary as a bytes-like object.\n" " is_raw\n" -" This parameter is for advanced user. True means dict_content\n" -" argument is a \"raw content\" dictionary, free of any format\n" -" restriction. False means dict_content argument is an ordinary\n" -" zstd dictionary, was created by zstd functions, follow a\n" -" specified format.\n" +" If true, perform no checks on *dict_content*, useful for some\n" +" advanced cases. Otherwise, check that the content represents\n" +" a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" -"ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be shared\n" +"by multiple ZstdCompressor or ZstdDecompressor objects."); -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw); +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, + int is_raw); -static int -_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -46,7 +44,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), }, + .ob_item = { &_Py_ID(is_raw), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -55,7 +53,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"dict_content", "is_raw", NULL}; + static const char * const _keywords[] = {"", "is_raw", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "ZstdDict", @@ -70,20 +68,20 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) int is_raw = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } dict_content = fastargs[0]; if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } is_raw = PyObject_IsTrue(fastargs[1]); if (is_raw < 0) { goto exit; } -skip_optional_pos: - return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw); +skip_optional_kwonly: + return_value = _zstd_ZstdDict_new_impl(type, dict_content, is_raw); exit: return return_value; @@ -204,4 +202,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 4f340f56b6fca8..97693fd043efb2 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdCompressor class definitions */ @@ -21,7 +18,6 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" #include "buffer.h" #include "zstddict.h" -#include // bool #include // offsetof() #include // ZSTD_*() @@ -42,9 +38,6 @@ typedef struct { /* Compression level */ int compression_level; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) @@ -122,7 +115,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, self->compression_level = value_v; } else if (key_v == ZSTD_c_nbWorkers) { - /* From zstd library doc: + /* From the zstd library docs: 1. When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream2(). 2, Default value is `0`, aka "single-threaded mode" : no @@ -189,8 +182,8 @@ _get_CDict(ZstdDict *self, int compressionLevel) _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_CDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_CDict instance from " + "Zstandard dictionary content."); } goto error; } @@ -318,20 +311,34 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) return 0; } +/*[clinic input] +@classmethod +_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new + level: object = None + The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT. + options: object = None + A dict object that contains advanced compression parameters. + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + +Create a compressor object for compressing data incrementally. + +Thread-safe at method level. For one-shot compression, use the compress() +function instead. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict) +/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/ { - ZstdCompressor *self; - self = PyObject_GC_New(ZstdCompressor, type); + ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } - self->initialized = 0; - self->dict = NULL; self->use_multithread = 0; - /* Compression context */ self->cctx = ZSTD_createCCtx(); if (self->cctx == NULL) { @@ -346,6 +353,37 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject /* Last mode */ self->last_mode = ZSTD_e_end; + if (level != Py_None && options != Py_None) { + PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); + goto error; + } + + /* Set compressLevel/options to compression context */ + if (level != Py_None) { + if (_zstd_set_c_parameters(self, level, "level", "int") < 0) { + goto error; + } + } + + if (options != Py_None) { + if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) { + goto error; + } + } + + /* Load Zstandard dictionary to compression context */ + self->dict = NULL; + if (zstd_dict != Py_None) { + if (_zstd_load_c_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: @@ -373,67 +411,6 @@ ZstdCompressor_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdCompressor.__init__ - - level: object = None - The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. - options: object = None - A dict object that contains advanced compression parameters. - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - -Create a compressor object for compressing data incrementally. - -Thread-safe at method level. For one-shot compression, use the compress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ -{ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; - } - self->initialized = 1; - - if (level != Py_None && options != Py_None) { - PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); - return -1; - } - - /* Set compressLevel/options to compression context */ - if (level != Py_None) { - if (_zstd_set_c_parameters(self, level, "level", "int") < 0) { - return -1; - } - } - - if (options != Py_None) { - if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) { - return -1; - } - } - - /* Load dictionary to compression context */ - if (zstd_dict != Py_None) { - if (_zstd_load_c_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - static PyObject * compress_impl(ZstdCompressor *self, Py_buffer *data, ZSTD_EndDirective end_directive) @@ -469,7 +446,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive); @@ -535,7 +512,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS do { @@ -644,14 +621,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since zstd data consists -of one or more independent frames, the compressor object can still be used -after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard data +consists of one or more independent frames, the compressor object can still +be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=0ab19627f323cdbc]*/ { PyObject *ret; @@ -719,10 +696,9 @@ ZstdCompressor_clear(PyObject *ob) static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, {Py_tp_dealloc, ZstdCompressor_dealloc}, - {Py_tp_init, _zstd_ZstdCompressor___init__}, {Py_tp_methods, ZstdCompressor_methods}, {Py_tp_members, ZstdCompressor_members}, - {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_clear, ZstdCompressor_clear}, {0, 0} diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index dc3e90a22d8dbb..852b796a872eef 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdDecompressor class definition */ @@ -48,9 +45,6 @@ typedef struct { /* For ZstdDecompressor, 0 or 1. 1 means the end of the first frame has been reached. */ bool eof; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) @@ -81,8 +75,8 @@ _get_DDict(ZstdDict *self) _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_DDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_DDict instance from " + "Zstandard dictionary content."); } } } @@ -265,8 +259,8 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) finish ZSTD_decompressStream()'s size_t return value: - - 0 when a frame is completely decoded and fully flushed, zstd's internal - buffer has no data. + - 0 when a frame is completely decoded and fully flushed, + zstd's internal buffer has no data. - An error code, which can be tested using ZSTD_isError(). - Or any other value > 0, which means there is still some decoding or flushing to do to complete current frame. @@ -311,7 +305,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, } /* Need to check out before in. Maybe zstd's internal buffer still has - a few bytes can be output, grow the buffer and continue. */ + a few bytes that can be output, grow the buffer and continue. */ if (out.pos == out.size) { /* Output buffer exhausted */ @@ -373,7 +367,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length /* Check .eof flag */ if (self->eof) { - PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame."); + PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame."); assert(ret == NULL); return NULL; } @@ -530,17 +524,30 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length } +/*[clinic input] +@classmethod +_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + options: object = None + A dict object that contains advanced decompression parameters. + +Create a decompressor object for decompressing data incrementally. + +Thread-safe at method level. For one-shot decompression, use the decompress() +function instead. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options) +/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/ { - ZstdDecompressor *self; - self = PyObject_GC_New(ZstdDecompressor, type); + ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } - self->initialized = 0; - self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; self->in_begin = -1; @@ -562,6 +569,26 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } + /* Load Zstandard dictionary to decompression context */ + self->dict = NULL; + if (zstd_dict != Py_None) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + /* Set option to decompression context */ + if (options != Py_None) { + if (_zstd_set_d_parameters(self, options) < 0) { + goto error; + } + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: @@ -595,55 +622,6 @@ ZstdDecompressor_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdDecompressor.__init__ - - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - options: object = None - A dict object that contains advanced decompression parameters. - -Create a decompressor object for decompressing data incrementally. - -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ -{ - /* Only called once */ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; - } - self->initialized = 1; - - /* Load dictionary to decompression context */ - if (zstd_dict != Py_None) { - if (_zstd_load_d_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - /* Set option to decompression context */ - if (options != Py_None) { - if (_zstd_set_d_parameters(self, options) < 0) { - return -1; - } - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - /*[clinic input] @critical_section @getter @@ -685,7 +663,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) _zstd.ZstdDecompressor.decompress data: Py_buffer - A bytes-like object, zstd data to be decompressed. + A bytes-like object, Zstandard data to be decompressed. max_length: Py_ssize_t = -1 Maximum size of returned data. When it is negative, the size of output buffer is unlimited. When it is nonnegative, returns at @@ -711,7 +689,7 @@ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/ { PyObject *ret; /* Thread-safe code */ @@ -769,11 +747,10 @@ ZstdDecompressor_clear(PyObject *ob) static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, {Py_tp_dealloc, ZstdDecompressor_dealloc}, - {Py_tp_init, _zstd_ZstdDecompressor___init__}, {Py_tp_methods, ZstdDecompressor_methods}, {Py_tp_members, ZstdDecompressor_members}, {Py_tp_getset, ZstdDecompressor_getset}, - {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 47bc8a84ca24a7..264946e8da2a99 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdDict class definitions */ @@ -25,17 +22,35 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #define ZstdDict_CAST(op) ((ZstdDict *)op) +/*[clinic input] +@classmethod +_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new + dict_content: object + The content of a Zstandard dictionary as a bytes-like object. + / + * + is_raw: bool = False + If true, perform no checks on *dict_content*, useful for some + advanced cases. Otherwise, check that the content represents + a Zstandard dictionary created by the zstd library or CLI. + +Represents a Zstandard dictionary. + +The dictionary can be used for compression or decompression, and can be shared +by multiple ZstdCompressor or ZstdDecompressor objects. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, + int is_raw) +/*[clinic end generated code: output=3ebff839cb3be6d7 input=6b5de413869ae878]*/ { - ZstdDict *self; - self = PyObject_GC_New(ZstdDict, type); + ZstdDict* self = PyObject_GC_New(ZstdDict, type); if (self == NULL) { goto error; } self->dict_content = NULL; - self->initialized = 0; self->d_dict = NULL; /* ZSTD_CDict dict */ @@ -44,6 +59,36 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U goto error; } + /* Check dict_content's type */ + self->dict_content = PyBytes_FromObject(dict_content); + if (self->dict_content == NULL) { + PyErr_SetString(PyExc_TypeError, + "dict_content argument should be bytes-like object."); + goto error; + } + + /* Both ordinary dictionary and "raw content" dictionary should + at least 8 bytes */ + if (Py_SIZE(self->dict_content) < 8) { + PyErr_SetString(PyExc_ValueError, + "Zstandard dictionary content should at least 8 bytes."); + goto error; + } + + /* Get dict_id, 0 means "raw content" dictionary. */ + self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content), + Py_SIZE(self->dict_content)); + + /* Check validity for ordinary dictionary */ + if (!is_raw && self->dict_id == 0) { + char *msg = "Invalid Zstandard dictionary and is_raw not set.\n"; + PyErr_SetString(PyExc_ValueError, msg); + goto error; + } + + // Can only track self once self->dict_content is included + PyObject_GC_Track(self); + return (PyObject*)self; error: @@ -72,83 +117,15 @@ ZstdDict_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdDict.__init__ - - dict_content: object - A bytes-like object, dictionary's content. - is_raw: bool = False - This parameter is for advanced user. True means dict_content - argument is a "raw content" dictionary, free of any format - restriction. False means dict_content argument is an ordinary - zstd dictionary, was created by zstd functions, follow a - specified format. - -Represents a zstd dictionary, which can be used for compression/decompression. - -It's thread-safe, and can be shared by multiple ZstdCompressor / -ZstdDecompressor objects. -[clinic start generated code]*/ - -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw) -/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ -{ - /* Only called once */ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; - } - self->initialized = 1; - - /* Check dict_content's type */ - self->dict_content = PyBytes_FromObject(dict_content); - if (self->dict_content == NULL) { - PyErr_SetString(PyExc_TypeError, - "dict_content argument should be bytes-like object."); - return -1; - } - - /* Both ordinary dictionary and "raw content" dictionary should - at least 8 bytes */ - if (Py_SIZE(self->dict_content) < 8) { - PyErr_SetString(PyExc_ValueError, - "Zstd dictionary content should at least 8 bytes."); - return -1; - } - - /* Get dict_id, 0 means "raw content" dictionary. */ - self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content), - Py_SIZE(self->dict_content)); - - /* Check validity for ordinary dictionary */ - if (!is_raw && self->dict_id == 0) { - char *msg = "The dict_content argument is not a valid zstd " - "dictionary. The first 4 bytes of a valid zstd dictionary " - "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n" - "If you are an advanced user, and can be sure that " - "dict_content argument is a \"raw content\" zstd " - "dictionary, set is_raw parameter to True."; - PyErr_SetString(PyExc_ValueError, msg); - return -1; - } - - // Can only track self once self->dict_content is included - PyObject_GC_Track(self); - return 0; -} - PyDoc_STRVAR(ZstdDict_dictid_doc, -"ID of zstd dictionary, a 32-bit unsigned int value.\n\n" -"Non-zero means ordinary dictionary, was created by zstd functions, follow\n" -"a specified format.\n\n" -"0 means a \"raw content\" dictionary, free of any format restriction, used\n" -"for advanced user."); +"the Zstandard dictionary, an int between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary, " +"conforming to the standardised format.\n\n" +"The special value '0' means a 'raw content' dictionary," +"without any restrictions on format or content."); PyDoc_STRVAR(ZstdDict_dictcontent_doc, -"The content of zstd dictionary, a bytes object, it's the same as dict_content\n" -"argument in ZstdDict.__init__() method. It can be used with other programs."); +"The content of a Zstandard dictionary, as a bytes object."); static PyObject * ZstdDict_str(PyObject *ob) @@ -266,9 +243,8 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_getset, ZstdDict_getset}, {Py_tp_new, _zstd_ZstdDict_new}, {Py_tp_dealloc, ZstdDict_dealloc}, - {Py_tp_init, _zstd_ZstdDict___init__}, {Py_tp_str, ZstdDict_str}, - {Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, {Py_tp_clear, ZstdDict_clear}, diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index e82498f5dd1901..e8a55a3670b869 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -1,12 +1,8 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef ZSTD_DICT_H #define ZSTD_DICT_H -#include // bool #include // ZSTD_DDict typedef struct { @@ -23,9 +19,6 @@ typedef struct { PyObject *dict_content; /* Dictionary id */ uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdDict; #endif // !ZSTD_DICT_H From 07a2033fcfabcbfb28450f3106b396b474561e57 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 04:58:51 +0200 Subject: [PATCH 143/713] [3.14] gh-132983: Call Py_XDECREF rather than PyObject_GC_Del in failed __new__ (GH-133962) (#134305) gh-132983: Call Py_XDECREF rather than PyObject_GC_Del in failed __new__ (GH-133962) Call Py_XDECREF rather than PyObject_GC_Del in failed __new__ This will call tp_dealloc and clear all members. (cherry picked from commit e575190abbd9409adad3e7fd95424f827236bed9) Co-authored-by: Petr Viktorin --- Modules/_zstd/compressor.c | 10 +++++----- Modules/_zstd/decompressor.c | 10 +++++----- Modules/_zstd/zstddict.c | 9 +++++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 97693fd043efb2..38baee2be1e95b 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -338,6 +338,7 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, } self->use_multithread = 0; + self->dict = NULL; /* Compression context */ self->cctx = ZSTD_createCCtx(); @@ -372,7 +373,6 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, } /* Load Zstandard dictionary to compression context */ - self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_c_dict(self, zstd_dict) < 0) { goto error; @@ -387,9 +387,7 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -401,7 +399,9 @@ ZstdCompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free compression context */ - ZSTD_freeCCtx(self->cctx); + if (self->cctx) { + ZSTD_freeCCtx(self->cctx); + } /* Py_XDECREF the dict after free the compression context */ Py_CLEAR(self->dict); diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 852b796a872eef..58f9c9f804e549 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -554,6 +554,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, self->in_end = -1; self->unused_data = NULL; self->eof = 0; + self->dict = NULL; /* needs_input flag */ self->needs_input = 1; @@ -570,7 +571,6 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, } /* Load Zstandard dictionary to decompression context */ - self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_d_dict(self, zstd_dict) < 0) { goto error; @@ -592,9 +592,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -606,7 +604,9 @@ ZstdDecompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free decompression context */ - ZSTD_freeDCtx(self->dctx); + if (self->dctx) { + ZSTD_freeDCtx(self->dctx); + } /* Py_CLEAR the dict after free decompression context */ Py_CLEAR(self->dict); diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 264946e8da2a99..7df187a6fa69d7 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -52,6 +52,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, self->dict_content = NULL; self->d_dict = NULL; + self->dict_id = 0; /* ZSTD_CDict dict */ self->c_dicts = PyDict_New(); @@ -92,9 +93,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -106,7 +105,9 @@ ZstdDict_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free ZSTD_DDict instance */ - ZSTD_freeDDict(self->d_dict); + if (self->d_dict) { + ZSTD_freeDDict(self->d_dict); + } /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ Py_CLEAR(self->dict_content); From 90aa13ae478f1eb1d5d02867382c33ee7670ab5c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 08:53:44 +0200 Subject: [PATCH 144/713] [3.14] gh-133374: fix test_python_legacy_windows_stdio (GH-134080) (GH-134314) (cherry picked from commit 652d6938ef8c42c1c4c180c3f0e257c26c6677da) Co-authored-by: Inada Naoki --- Lib/test/test_cmd_line.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1b40e0d05fe3bc..f540973c11e01c 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -972,10 +972,25 @@ def test_python_legacy_windows_fs_encoding(self): @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') def test_python_legacy_windows_stdio(self): - code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" - expected = 'cp' - rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') - self.assertIn(expected.encode(), out) + # Test that _WindowsConsoleIO is used when PYTHONLEGACYWINDOWSSTDIO + # is not set. + # We cannot use PIPE becase it prevents creating new console. + # So we use exit code. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != '_WindowsConsoleIO')" + env = os.environ.copy() + env["PYTHONLEGACYWINDOWSSTDIO"] = "" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) + + # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != 'FileIO')" + env["PYTHONLEGACYWINDOWSSTDIO"] = "1" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + self.assertEqual(p.returncode, 0) @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") From 9be3413b60d8fdcc0fed605440815e0532932a39 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 10:18:13 +0200 Subject: [PATCH 145/713] [3.14] Clean up test_posixpath (GH-134315) (GH-134316) * Ensure that created files and dirs are always removed after test. Now addCleanup() does not conflict with tearDown(). * Use os_helper.unlink() and os_helper.rmdir(). * Import TESTFN from os_helper. (cherry picked from commit e29171bf8a26b5faf97222e07a7d5f33c9eb272b) Co-authored-by: Serhiy Storchaka --- Lib/test/test_posixpath.py | 86 ++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index fa19d549c26cfa..572ffcf1aae36f 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -9,7 +9,7 @@ from test import test_genericpath from test.support import import_helper from test.support import os_helper -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, TESTFN from unittest import mock try: @@ -21,7 +21,7 @@ # An absolute path to a temporary filename for testing. We can't rely on TESTFN # being an absolute path, so we need this. -ABSTFN = abspath(os_helper.TESTFN) +ABSTFN = abspath(TESTFN) def skip_if_ABSTFN_contains_backslash(test): """ @@ -33,21 +33,11 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass - class PosixPathTest(unittest.TestCase): def setUp(self): - self.tearDown() - - def tearDown(self): for suffix in ["", "1", "2"]: - os_helper.unlink(os_helper.TESTFN + suffix) - safe_rmdir(os_helper.TESTFN + suffix) + self.assertFalse(posixpath.lexists(ABSTFN + suffix)) def test_join(self): fn = posixpath.join @@ -194,25 +184,28 @@ def test_dirname(self): self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") def test_islink(self): - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), False) - with open(os_helper.TESTFN + "1", "wb") as f: + self.addCleanup(os_helper.unlink, TESTFN + "1") + with open(TESTFN + "1", "wb") as f: f.write(b"foo") - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) if os_helper.can_symlink(): - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN + "2") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - os.remove(os_helper.TESTFN + "1") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - self.assertIs(posixpath.exists(os_helper.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), True) - - self.assertIs(posixpath.islink(os_helper.TESTFN + "\udfff"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\xff"), False) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\x00"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\x00"), False) + self.addCleanup(os_helper.unlink, TESTFN + "2") + os.symlink(TESTFN + "1", TESTFN + "2") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + os.remove(TESTFN + "1") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + self.assertIs(posixpath.exists(TESTFN + "2"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), True) + + def test_islink_invalid_paths(self): + self.assertIs(posixpath.islink(TESTFN + "\udfff"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\xff"), False) + self.assertIs(posixpath.islink(TESTFN + "\x00"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\x00"), False) def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) @@ -227,7 +220,7 @@ def test_ismount_non_existent(self): os.mkdir(ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) @@ -241,7 +234,7 @@ def test_ismount_symlinks(self): os.symlink("/", ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @unittest.skipIf(posix is None, "Test requires posix module") def test_ismount_different_device(self): @@ -502,10 +495,10 @@ def test_realpath_relative(self): @skip_if_ABSTFN_contains_backslash def test_realpath_missing_pardir(self): try: - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) - self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + os.symlink(TESTFN + "1", TESTFN) + self.assertEqual(realpath("nonexistent/../" + TESTFN), ABSTFN + "1") finally: - os_helper.unlink(os_helper.TESTFN) + os_helper.unlink(TESTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -601,7 +594,7 @@ def test_realpath_repeated_indirect_symlinks(self): finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -620,7 +613,7 @@ def test_realpath_deep_recursion(self): finally: for i in range(depth + 1): os_helper.unlink(ABSTFN + '/%d' % i) - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -638,8 +631,8 @@ def test_realpath_resolve_parents(self): self.assertEqual(realpath("a"), ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/y") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -665,9 +658,9 @@ def test_realpath_resolve_before_normalizing(self): ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k/y") + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -685,8 +678,8 @@ def test_realpath_resolve_first(self): self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -704,7 +697,7 @@ def test_realpath_unreadable_symlink(self): realpath(ABSTFN, strict=True) finally: os.chmod(ABSTFN, 0o755, follow_symlinks=False) - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @skip_if_ABSTFN_contains_backslash def test_realpath_nonterminal_file(self): @@ -743,6 +736,7 @@ def test_realpath_nonterminal_symlink_to_file(self): self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -764,6 +758,8 @@ def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + os_helper.unlink(ABSTFN + "2") def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") @@ -889,8 +885,8 @@ class PathLikeTests(unittest.TestCase): path = posixpath def setUp(self): - self.file_name = os_helper.TESTFN - self.file_path = FakePath(os_helper.TESTFN) + self.file_name = TESTFN + self.file_path = FakePath(TESTFN) self.addCleanup(os_helper.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") From ccaf865364692087dc74252dd411f6356a1d43f0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 11:57:39 +0200 Subject: [PATCH 146/713] [3.14] gh-131505: Move len boundary assertions before using len. (GH-131536) (GH-134239) gh-131505: Move len boundary assertions before using len. (GH-131536) Move len boundary assertions before using len. (cherry picked from commit c45e661226558e997e265cf53ce1419213cc10b7) Co-authored-by: naya451 <41294408+naya451@users.noreply.github.com> --- Modules/_io/bytesio.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e45a2d1a16dcba..8bc7aeacf294ba 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -583,9 +583,9 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) len = 0; } - memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); assert(self->pos + len < PY_SSIZE_T_MAX); assert(len >= 0); + memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; return PyLong_FromSsize_t(len); From 9c73a4de0c6f593cfa53afdd21d4aa8da1b15d46 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 12:11:39 +0200 Subject: [PATCH 147/713] [3.14] gh-133940: test_strftime incorrectly calculates expected week (GH-134281) (#134301) gh-133940: test_strftime incorrectly calculates expected week (GH-134281) Let the system determine the correct tm_wday and tm_isdst. (cherry picked from commit e3dda8f81832008adf19906004f0cd53de95dd0b) Co-authored-by: Gustaf <79180496+GGyll@users.noreply.github.com> --- Lib/test/test_strftime.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 752e31359cf206..375f6aaedd8934 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -39,7 +39,21 @@ def _update_variables(self, now): if now[3] < 12: self.ampm='(AM|am)' else: self.ampm='(PM|pm)' - self.jan1 = time.localtime(time.mktime((now[0], 1, 1, 0, 0, 0, 0, 1, 0))) + jan1 = time.struct_time( + ( + now.tm_year, # Year + 1, # Month (January) + 1, # Day (1st) + 0, # Hour (0) + 0, # Minute (0) + 0, # Second (0) + -1, # tm_wday (will be determined) + 1, # tm_yday (day 1 of the year) + -1, # tm_isdst (let the system determine) + ) + ) + # use mktime to get the correct tm_wday and tm_isdst values + self.jan1 = time.localtime(time.mktime(jan1)) try: if now[8]: self.tz = time.tzname[1] From c23bdc2eced311671d77c5ac2cb450fe4eb1b08e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 12:13:40 +0200 Subject: [PATCH 148/713] [3.14] gh-133890: Handle UnicodeEncodeError in tarfile (GH-134147) (GH-134195) UnicodeEncodeError is now handled the same way as OSError during TarFile member extraction. (cherry picked from commit 9983c7d4416cac8deb2fded1ec9c7daf786c3a02) Co-authored-by: Serhiy Storchaka --- Lib/tarfile.py | 4 +- Lib/test/test_tarfile.py | 49 +++++++++++++++++-- ...-05-17-18-08-35.gh-issue-133890.onn9_X.rst | 2 + 3 files changed, 49 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-17-18-08-35.gh-issue-133890.onn9_X.rst diff --git a/Lib/tarfile.py b/Lib/tarfile.py index 13889d768021b1..212b71f6509740 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -2439,7 +2439,7 @@ def _get_extract_tarinfo(self, member, filter_function, path): unfiltered = tarinfo try: tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: + except (OSError, UnicodeEncodeError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) @@ -2460,7 +2460,7 @@ def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, numeric_owner=numeric_owner) - except OSError as e: + except (OSError, UnicodeEncodeError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2d9649237a9382..2018a20afd1b18 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -3490,11 +3490,12 @@ class ArchiveMaker: with t.open() as tar: ... # `tar` is now a TarFile with 'filename' in it! """ - def __init__(self): + def __init__(self, **kwargs): self.bio = io.BytesIO() + self.tar_kwargs = dict(kwargs) def __enter__(self): - self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio) + self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs) return self def __exit__(self, *exc): @@ -4073,7 +4074,10 @@ def test_tar_filter(self): # that in the test archive.) with tarfile.TarFile.open(tarname) as tar: for tarinfo in tar.getmembers(): - filtered = tarfile.tar_filter(tarinfo, '') + try: + filtered = tarfile.tar_filter(tarinfo, '') + except UnicodeEncodeError: + continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) @@ -4084,11 +4088,48 @@ def test_data_filter(self): for tarinfo in tar.getmembers(): try: filtered = tarfile.data_filter(tarinfo, '') - except tarfile.FilterError: + except (tarfile.FilterError, UnicodeEncodeError): continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_filter_unencodable(self): + # Sanity check using a valid path. + tarinfo = tarfile.TarInfo(os_helper.TESTFN) + filtered = tarfile.tar_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + filtered = tarfile.data_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + + tarinfo = tarfile.TarInfo('test\x00') + self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '') + tarinfo = tarfile.TarInfo('\ud800') + self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '') + + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_extract_unencodable(self): + # Create a member with name \xed\xa0\x80 which is UTF-8 encoded + # lone surrogate \ud800. + with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc: + arc.add('\udced\udca0\udc80') + with os_helper.temp_cwd() as tmp: + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=1) + self.assertEqual(tar.getnames(), ['\ud800']) + with self.assertRaises(UnicodeEncodeError): + tar.extractall() + self.assertEqual(os.listdir(), []) + + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=0, debug=1) + with support.captured_stderr() as stderr: + tar.extractall() + self.assertEqual(os.listdir(), []) + self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue()) + def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): diff --git a/Misc/NEWS.d/next/Library/2025-05-17-18-08-35.gh-issue-133890.onn9_X.rst b/Misc/NEWS.d/next/Library/2025-05-17-18-08-35.gh-issue-133890.onn9_X.rst new file mode 100644 index 00000000000000..44565a5424e65b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-17-18-08-35.gh-issue-133890.onn9_X.rst @@ -0,0 +1,2 @@ +The :mod:`tarfile` module now handles :exc:`UnicodeEncodeError` in the same +way as :exc:`OSError` when cannot extract a member. From a3fa72578a68fc4a3caf27b530150cbb7bad8793 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 14:15:34 +0200 Subject: [PATCH 149/713] [3.14] gh-53189: Document peculiarities of InteractiveConsole in relation to pickle (GH-123069) (GH-134328) gh-53189: Document peculiarities of InteractiveConsole in relation to pickle (GH-123069) (cherry picked from commit a31bbc951a9d74cd7b9092555c101e51a2b9482b) Co-authored-by: Serhiy Storchaka Co-authored-by: Terry Jan Reedy --- Doc/library/code.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8f7692df9fb22d..52587c4dd8f8e8 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -22,6 +22,12 @@ build applications which provide an interactive interpreter prompt. it defaults to a newly created dictionary with key ``'__name__'`` set to ``'__console__'`` and key ``'__doc__'`` set to ``None``. + Note that functions and classes objects created under an + :class:`!InteractiveInterpreter` instance will belong to the namespace + specified by *locals*. + They are only pickleable if *locals* is the namespace of an existing + module. + .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) From 7e67b36d977f480b2cc28fbab671c119d60bc11a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 14:55:21 +0200 Subject: [PATCH 150/713] [3.14] gh-117596: Add more tests for os.path with invalid paths (GH-134189) (GH-134265) (cherry picked from commit 871d26987533e81ab63af067e1fc96aa37a26bf7) Co-authored-by: Serhiy Storchaka --- Lib/test/test_ntpath.py | 179 ++++++++++++++++++++++++++++++++++--- Lib/test/test_posixpath.py | 74 +++++++++++++++ 2 files changed, 243 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index c10387b58e3f9c..f83ef225a6e48e 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -124,6 +124,22 @@ def test_splitdrive(self): tester('ntpath.splitdrive("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/dir")) + def test_splitdrive_invalid_paths(self): + splitdrive = ntpath.splitdrive + self.assertEqual(splitdrive('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\di\x00r')) + self.assertEqual(splitdrive(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\di\x00r')) + self.assertEqual(splitdrive("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitdrive(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\\xfd')) + def test_splitroot(self): tester("ntpath.splitroot('')", ('', '', '')) tester("ntpath.splitroot('foo')", ('', '', 'foo')) @@ -214,6 +230,22 @@ def test_splitroot(self): tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_splitroot_invalid_paths(self): + splitroot = ntpath.splitroot + self.assertEqual(splitroot('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\', 'di\x00r')) + self.assertEqual(splitroot(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\', b'di\x00r')) + self.assertEqual(splitroot("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\', '\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitroot(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\', b'\xfd')) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', @@ -226,6 +258,21 @@ def test_split(self): tester('ntpath.split("c:/")', ('c:/', '')) tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) + def test_split_invalid_paths(self): + split = ntpath.split + self.assertEqual(split('c:\\fo\x00o\\ba\x00r'), + ('c:\\fo\x00o', 'ba\x00r')) + self.assertEqual(split(b'c:\\fo\x00o\\ba\x00r'), + (b'c:\\fo\x00o', b'ba\x00r')) + self.assertEqual(split('c:\\\udfff\\\udffe'), + ('c:\\\udfff', '\udffe')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, split, b'c:\\\xff\\bar') + self.assertRaises(UnicodeDecodeError, split, b'c:\\foo\\\xff') + else: + self.assertEqual(split(b'c:\\\xff\\\xfe'), + (b'c:\\\xff', b'\xfe')) + def test_isabs(self): tester('ntpath.isabs("foo\\bar")', 0) tester('ntpath.isabs("foo/bar")', 0) @@ -333,6 +380,30 @@ def test_join(self): tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') + def test_normcase(self): + normcase = ntpath.normcase + self.assertEqual(normcase(''), '') + self.assertEqual(normcase(b''), b'') + self.assertEqual(normcase('ABC'), 'abc') + self.assertEqual(normcase(b'ABC'), b'abc') + self.assertEqual(normcase('\xc4\u0141\u03a8'), '\xe4\u0142\u03c8') + expected = '\u03c9\u2126' if sys.platform == 'win32' else '\u03c9\u03c9' + self.assertEqual(normcase('\u03a9\u2126'), expected) + if sys.platform == 'win32' or sys.getfilesystemencoding() == 'utf-8': + self.assertEqual(normcase('\xc4\u0141\u03a8'.encode()), + '\xe4\u0142\u03c8'.encode()) + self.assertEqual(normcase('\u03a9\u2126'.encode()), + expected.encode()) + + def test_normcase_invalid_paths(self): + normcase = ntpath.normcase + self.assertEqual(normcase('abc\x00def'), 'abc\x00def') + self.assertEqual(normcase(b'abc\x00def'), b'abc\x00def') + self.assertEqual(normcase('\udfff'), '\udfff') + if sys.platform == 'win32': + path = b'ABC' + bytes(range(128, 256)) + self.assertEqual(normcase(path), path.lower()) + def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') tester("ntpath.normpath('A/./B')", r'A\B') @@ -381,6 +452,21 @@ def test_normpath(self): tester("ntpath.normpath('\\\\')", '\\\\') tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') + def test_normpath_invalid_paths(self): + normpath = ntpath.normpath + self.assertEqual(normpath('fo\x00o'), 'fo\x00o') + self.assertEqual(normpath(b'fo\x00o'), b'fo\x00o') + self.assertEqual(normpath('fo\x00o\\..\\bar'), 'bar') + self.assertEqual(normpath(b'fo\x00o\\..\\bar'), b'bar') + self.assertEqual(normpath('\udfff'), '\udfff') + self.assertEqual(normpath('\udfff\\..\\foo'), 'foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, normpath, b'\xff') + self.assertRaises(UnicodeDecodeError, normpath, b'\xff\\..\\foo') + else: + self.assertEqual(normpath(b'\xff'), b'\xff') + self.assertEqual(normpath(b'\xff\\..\\foo'), b'foo') + def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('.')", expected) @@ -420,10 +506,6 @@ def test_realpath_basic(self): d = drives.pop().encode() self.assertEqual(ntpath.realpath(d), d) - # gh-106242: Embedded nulls and non-strict fallback to abspath - self.assertEqual(ABSTFN + "\0spam", - ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) - @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_strict(self): @@ -434,8 +516,51 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_invalid_paths(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFN + '\x00' + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(realpath(path, strict=False), path) # gh-106242: Embedded nulls should raise OSError (not ValueError) - self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) + self.assertRaises(OSError, realpath, path, strict=True) + path = ABSTFNb + b'\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + path = ABSTFN + '\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + path = ABSTFNb + b'\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + path = ABSTFN + '\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwd()) + self.assertEqual(realpath(path, strict=True), os.getcwd()) + path = ABSTFNb + b'\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwdb()) + self.assertEqual(realpath(path, strict=True), os.getcwdb()) + path = ABSTFN + '\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + path = ABSTFNb + b'\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + + path = ABSTFNb + b'\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + path = ABSTFNb + b'\\nonexistent\\\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + path = ABSTFNb + b'\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + path = ABSTFNb + b'\\nonexistent\\\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -812,8 +937,6 @@ def test_abspath(self): tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) - self.assertEqual(ntpath.abspath("C:\x00"), ntpath.join(ntpath.abspath("C:"), "\x00")) - self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -847,6 +970,26 @@ def test_abspath(self): drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") + def test_abspath_invalid_paths(self): + abspath = ntpath.abspath + if sys.platform == 'win32': + self.assertEqual(abspath("C:\x00"), ntpath.join(abspath("C:"), "\x00")) + self.assertEqual(abspath(b"C:\x00"), ntpath.join(abspath(b"C:"), b"\x00")) + self.assertEqual(abspath("\x00:spam"), "\x00:\\spam") + self.assertEqual(abspath(b"\x00:spam"), b"\x00:\\spam") + self.assertEqual(abspath('c:\\fo\x00o'), 'c:\\fo\x00o') + self.assertEqual(abspath(b'c:\\fo\x00o'), b'c:\\fo\x00o') + self.assertEqual(abspath('c:\\fo\x00o\\..\\bar'), 'c:\\bar') + self.assertEqual(abspath(b'c:\\fo\x00o\\..\\bar'), b'c:\\bar') + self.assertEqual(abspath('c:\\\udfff'), 'c:\\\udfff') + self.assertEqual(abspath('c:\\\udfff\\..\\foo'), 'c:\\foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff') + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff\\..\\foo') + else: + self.assertEqual(abspath(b'c:\\\xff'), b'c:\\\xff') + self.assertEqual(abspath(b'c:\\\xff\\..\\foo'), b'c:\\foo') + def test_relpath(self): tester('ntpath.relpath("a")', 'a') tester('ntpath.relpath(ntpath.abspath("a"))', 'a') @@ -989,6 +1132,18 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_ismount_invalid_paths(self): + ismount = ntpath.ismount + self.assertFalse(ismount("c:\\\udfff")) + if sys.platform == 'win32': + self.assertRaises(ValueError, ismount, "c:\\\x00") + self.assertRaises(ValueError, ismount, b"c:\\\x00") + self.assertRaises(UnicodeDecodeError, ismount, b"c:\\\xff") + else: + self.assertFalse(ismount("c:\\\x00")) + self.assertFalse(ismount(b"c:\\\x00")) + self.assertFalse(ismount(b"c:\\\xff")) + def test_isreserved(self): self.assertFalse(ntpath.isreserved('')) self.assertFalse(ntpath.isreserved('.')) @@ -1095,6 +1250,13 @@ def test_isjunction(self): self.assertFalse(ntpath.isjunction('tmpdir')) self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + def test_isfile_invalid_paths(self): + isfile = ntpath.isfile + self.assertIs(isfile('/tmp\udfffabcds'), False) + self.assertIs(isfile(b'/tmp\xffabcds'), False) + self.assertIs(isfile('/tmp\x00abcds'), False) + self.assertIs(isfile(b'/tmp\x00abcds'), False) + @unittest.skipIf(sys.platform != 'win32', "drive letters are a windows concept") def test_isfile_driveletter(self): drive = os.environ.get('SystemDrive') @@ -1195,9 +1357,6 @@ def _check_function(self, func): def test_path_normcase(self): self._check_function(self.path.normcase) - if sys.platform == 'win32': - self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') - self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 572ffcf1aae36f..f3f9895f529470 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -222,6 +222,7 @@ def test_ismount_non_existent(self): finally: os_helper.rmdir(ABSTFN) + def test_ismount_invalid_paths(self): self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) self.assertIs(posixpath.ismount('/\x00'), False) @@ -482,6 +483,79 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_relative(self): From 2dedf5eb4523014192927fbeb132f93204d3a5f9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 16:05:16 +0200 Subject: [PATCH 151/713] [3.14] gh-62824: Add alias for iso-8859-8-i which is the same as iso-8859-8 (gh-134306) (gh-134330) (cherry picked from commit 5ab66a882d1b5e44ec50b25df116ab209d65863f) Co-authored-by: Bas Bloemsaat Co-authored-by: David Goncalves Co-authored-by: Oleg Iarygin --- Lib/encodings/aliases.py | 2 ++ Lib/test/test_codecs.py | 8 ++++++++ .../Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst | 1 + 3 files changed, 11 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index a94bb270671e37..4ecb6b6e297a13 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -405,6 +405,8 @@ 'iso_8859_8' : 'iso8859_8', 'iso_8859_8_1988' : 'iso8859_8', 'iso_ir_138' : 'iso8859_8', + 'iso_8859_8_i' : 'iso8859_8', + 'iso_8859_8_e' : 'iso8859_8', # iso8859_9 codec 'csisolatin5' : 'iso8859_9', diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index d42270da15ee32..1d6136405f428c 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3,6 +3,7 @@ import copy import io import pickle +import os import sys import unittest import encodings @@ -3107,6 +3108,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_file = os.path.join(encodings_dir, value + ".py") + self.assertTrue(os.path.isfile(codec_file), + "Codec file not found: " + codec_file) + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") diff --git a/Misc/NEWS.d/next/Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst b/Misc/NEWS.d/next/Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst new file mode 100644 index 00000000000000..1fe4e47c9ec213 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst @@ -0,0 +1 @@ +Fix aliases for ``iso8859_8`` encoding. Patch by Dave Goncalves. From b9a7e79ee1fdbcfdf6b086fc5323063bb4eaad92 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 16:15:18 +0200 Subject: [PATCH 152/713] [3.14] gh-128307: Update what's new in 3.13 and 3.14 with create_task changes of asyncio (GH-134304) (#134319) gh-128307: Update what's new in 3.13 and 3.14 with create_task changes of asyncio (GH-134304) (cherry picked from commit 28625d4f956f8d30671aba1daaac9735932983db) Co-authored-by: Guido van Rossum Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/whatsnew/3.13.rst | 16 ++++++++++++++++ Doc/whatsnew/3.14.rst | 18 ++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e20e49325c01d5..52bafeb4df7d41 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -730,6 +730,22 @@ asyncio never awaited). (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) +* The function and methods named ``create_task`` have received a new + ``**kwargs`` argument that is passed through to the task constructor. + This change was accidentally added in 3.13.3, + and broke the API contract for custom task factories. + Several third-party task factories implemented workarounds for this. + In 3.13.4 and later releases the old factory contract is honored + once again (until 3.14). + To keep the workarounds working, the extra ``**kwargs`` argument still + allows passing additional keyword arguments to :class:`~asyncio.Task` + and to custom task factories. + + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) base64 ------ diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 8f39b99e38e3a5..b192615e173f11 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1064,6 +1064,24 @@ ast (Contributed by Semyon Moroz in :gh:`133367`.) +asyncio +------- + +* The function and methods named :func:`!create_task` now take an arbitrary + list of keyword arguments. All keyword arguments are passed to the + :class:`~asyncio.Task` constructor or the custom task factory. + (See :meth:`~asyncio.loop.set_task_factory` for details.) + The ``name`` and ``context`` keyword arguments are no longer special; + the name should now be set using the ``name`` keyword argument of the factory, + and ``context`` may be ``None``. + + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) + + bdb --- From ff0facd33201ae0f4101e73b818b9af15d384455 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 16:52:11 +0200 Subject: [PATCH 153/713] [3.14] gh-101100: Fix Sphinx warnings in `library/decimal.rst` (GH-134303) (#134338) Co-authored-by: Yuki Kobayashi --- Doc/conf.py | 1 - Doc/library/decimal.rst | 72 ++++++++++++++++++++++++----------------- Doc/tools/.nitignore | 1 - 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/Doc/conf.py b/Doc/conf.py index 467961dd5e2bff..7fadad66cb3238 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -308,7 +308,6 @@ ('py:attr', '__annotations__'), ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), - ('py:attr', 'decimal.Context.clamp'), ('py:meth', 'index'), # list.index, tuple.index, etc. ] diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index f53c1f3467034e..10ddfa02b43156 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2,7 +2,7 @@ ===================================================================== .. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. + :synopsis: Implementation of the General Decimal Arithmetic Specification. .. moduleauthor:: Eric Price .. moduleauthor:: Facundo Batista @@ -121,7 +121,7 @@ reset them before monitoring a calculation. .. _decimal-tutorial: -Quick-start Tutorial +Quick-start tutorial -------------------- The usual start to using decimals is importing the module, viewing the current @@ -1096,40 +1096,52 @@ In addition to the three supplied contexts, new contexts can be created with the default values are copied from the :const:`DefaultContext`. If the *flags* field is not specified or is :const:`None`, all flags are cleared. - *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. + .. attribute:: prec - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. + An integer in the range [``1``, :const:`MAX_PREC`] that sets + the precision for arithmetic operations in the context. - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. + .. attribute:: rounding - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], - *Emax* in the range [``0``, :const:`MAX_EMAX`]. + One of the constants listed in the section `Rounding Modes`_. - The *capitals* field is either ``0`` or ``1`` (the default). If set to - ``1``, exponents are printed with a capital ``E``; otherwise, a - lowercase ``e`` is used: ``Decimal('6.02e+23')``. + .. attribute:: traps + flags - The *clamp* field is either ``0`` (the default) or ``1``. - If set to ``1``, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - ``0`` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is - ``1``, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: + Lists of any signals to be set. Generally, new contexts should only set + traps and leave the flags clear. - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') + .. attribute:: Emin + Emax - A *clamp* value of ``1`` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. + Integers specifying the outer limits allowable for exponents. *Emin* must + be in the range [:const:`MIN_EMIN`, ``0``], *Emax* in the range + [``0``, :const:`MAX_EMAX`]. + + .. attribute:: capitals + + Either ``0`` or ``1`` (the default). If set to + ``1``, exponents are printed with a capital ``E``; otherwise, a + lowercase ``e`` is used: ``Decimal('6.02e+23')``. + + .. attribute:: clamp + + Either ``0`` (the default) or ``1``. If set to ``1``, the exponent ``e`` + of a :class:`Decimal` instance representable in this context is strictly + limited to the range ``Emin - prec + 1 <= e <= Emax - prec + 1``. + If *clamp* is ``0`` then a weaker condition holds: the adjusted exponent of + the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is + ``1``, a large normal number will, where possible, have its + exponent reduced and a corresponding number of zeros added to its + coefficient, in order to fit the exponent constraints; this + preserves the value of the number but loses information about + significant trailing zeros. For example:: + + >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') + Decimal('1.23000E+999') + + A *clamp* value of ``1`` allows compatibility with the + fixed-width decimal interchange formats specified in IEEE 754. The :class:`Context` class defines several general purpose methods as well as a large number of methods for doing arithmetic directly in a given context. @@ -1769,7 +1781,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating-Point Notes +Floating-point notes -------------------- diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 926d2cd42bdf02..e3bcb968128d96 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -14,7 +14,6 @@ Doc/c-api/typeobj.rst Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst -Doc/library/decimal.rst Doc/library/email.charset.rst Doc/library/email.compat32-message.rst Doc/library/email.parser.rst From dc5866ab250f641df6ed78c943b0d36d6a4c04c2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 18:25:04 +0200 Subject: [PATCH 154/713] [3.14] gh-129748: Update mimalloc to use atomic store for mi_block_set_nextx (GH-134238) (gh-134352) gh-129748: Update mimalloc to use atomic store for mi_block_set_nextx (GH-134238) (cherry picked from commit 317c49622397222b7c7fb49837e6b1fd7e82a80d) Co-authored-by: Donghee Na --- Include/internal/mimalloc/mimalloc/internal.h | 8 ++++---- Include/internal/mimalloc/mimalloc/types.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index d97f51b8eefbe5..71b7ea702d6c5e 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -634,10 +634,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #ifdef MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed(&block->next), keys); #else MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)block->next; + next = (mi_block_t*)mi_atomic_load_relaxed(&block->next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; @@ -646,10 +646,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { mi_track_mem_undefined(block,sizeof(mi_block_t)); #ifdef MI_ENCODE_FREELIST - block->next = mi_ptr_encode(null, next, keys); + mi_atomic_store_relaxed(&block->next, mi_ptr_encode(null, next, keys)); #else MI_UNUSED(keys); MI_UNUSED(null); - block->next = (mi_encoded_t)next; + mi_atomic_store_relaxed(&block->next, (mi_encoded_t)next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); } diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 354839ba955b36..4f77bd7bc525db 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -235,7 +235,7 @@ typedef size_t mi_threadid_t; // free lists contain blocks typedef struct mi_block_s { - mi_encoded_t next; + _Atomic(mi_encoded_t) next; } mi_block_t; From 8e8d5c91cd9e161a74463ebbee7a1a563d12b637 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 18:46:52 +0200 Subject: [PATCH 155/713] [3.14] gh-132542: Set native thread ID after fork (GH-132701) (GH-134356) (cherry picked from commit 6b735023132a4ac9dc5b849d982104eeb1e8bdad) Co-authored-by: Noam Cohen --- Lib/test/test_threading.py | 28 +++++++++++++++++++ Lib/threading.py | 2 ++ ...-04-19-17-16-46.gh-issue-132542.7T_TY_.rst | 2 ++ 3 files changed, 32 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-17-16-46.gh-issue-132542.7T_TY_.rst diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index abe63c10c0ac7c..dc7172db04c5b7 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1352,6 +1352,34 @@ def do_flush(*args, **kwargs): ''') assert_python_ok("-c", script) + @skip_unless_reliable_fork + def test_native_id_after_fork(self): + script = """if True: + import threading + import os + from test import support + + parent_thread_native_id = threading.current_thread().native_id + print(parent_thread_native_id, flush=True) + assert parent_thread_native_id == threading.get_native_id() + childpid = os.fork() + if childpid == 0: + print(threading.current_thread().native_id, flush=True) + assert threading.current_thread().native_id == threading.get_native_id() + else: + try: + assert parent_thread_native_id == threading.current_thread().native_id + assert parent_thread_native_id == threading.get_native_id() + finally: + support.wait_process(childpid, exitcode=0) + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + native_ids = out.strip().splitlines() + self.assertEqual(len(native_ids), 2) + self.assertNotEqual(native_ids[0], native_ids[1]) + class ThreadJoinOnShutdown(BaseTestCase): def _run_and_join(self, script): diff --git a/Lib/threading.py b/Lib/threading.py index 39a1a7f4cdfda0..fa290d17c635ab 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -951,6 +951,8 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident assert self._os_thread_handle.ident == new_ident + if _HAVE_THREAD_NATIVE_ID: + self._set_native_id() else: # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() # already marked our handle done. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-17-16-46.gh-issue-132542.7T_TY_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-17-16-46.gh-issue-132542.7T_TY_.rst new file mode 100644 index 00000000000000..c69ce5efdedcca --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-17-16-46.gh-issue-132542.7T_TY_.rst @@ -0,0 +1,2 @@ +Update :attr:`Thread.native_id ` after +:manpage:`fork(2)` to ensure accuracy. Patch by Noam Cohen. From 402d983f27fd37ff6530ef9f04ad3e66402e4173 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 18:47:42 +0200 Subject: [PATCH 156/713] [3.14] gh-128002: add what's new docs for `asyncio` (GH-134324) (#134362) gh-128002: add what's new docs for `asyncio` (GH-134324) (cherry picked from commit 0584533dc70b4efdb65fd53fbb5beddbc7a302a8) Co-authored-by: Kumar Aditya --- Doc/whatsnew/3.14.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b192615e173f11..604f3365ebffcc 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1997,11 +1997,19 @@ Optimizations asyncio ------- -* :mod:`asyncio` now uses double linked list implementation for native tasks - which speeds up execution by 10% on standard pyperformance benchmarks and - reduces memory usage. +* :mod:`asyncio` has a new per-thread double linked list implementation internally for + :class:`native tasks ` which speeds up execution by 10-20% on standard + pyperformance benchmarks and reduces memory usage. + This enables external introspection tools such as + :ref:`python -m asyncio pstree ` + to introspect the call graph of asyncio tasks running in all threads. (Contributed by Kumar Aditya in :gh:`107803`.) +* :mod:`asyncio` has first class support for :term:`free-threading builds `. + This enables parallel execution of multiple event loops across different threads and scales + linearly with the number of threads. + (Contributed by Kumar Aditya in :gh:`128002`.) + * :mod:`asyncio` has new utility functions for introspecting and printing the program's call graph: :func:`asyncio.capture_call_graph` and :func:`asyncio.print_call_graph`. @@ -2083,7 +2091,6 @@ Deprecated * :class:`asyncio.WindowsProactorEventLoopPolicy` * :func:`asyncio.get_event_loop_policy` * :func:`asyncio.set_event_loop_policy` - * :func:`asyncio.set_event_loop` Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with *loop_factory* to use the desired event loop implementation. From 6d4b56d2f3501401d7ce86c3fd9002ee878d5a1d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 19:02:55 +0200 Subject: [PATCH 157/713] [3.14] gh-127945: Update What's New in Python 3.14 for free-threaded ctypes (GH-134332) (#134364) gh-127945: Update What's New in Python 3.14 for free-threaded ctypes (GH-134332) (cherry picked from commit b430e92dd80105e97b945a78a48bce13564bf843) Co-authored-by: Kumar Aditya --- Doc/whatsnew/3.14.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 604f3365ebffcc..88e52015bdc2b1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1200,6 +1200,9 @@ ctypes making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) +* :mod:`ctypes` now supports :term:`free-threading builds `. + (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + curses ------ From 6074e1e6c4be0b12c6c6b9af95ac5673a60a8c22 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 19:11:01 +0200 Subject: [PATCH 158/713] [3.14] gh-75459: Doc: C API: Improve object life cycle documentation (GH-125962) (GH-134344) gh-75459: Doc: C API: Improve object life cycle documentation (GH-125962) * Add "cyclic isolate" to the glossary. * Add a new "Object Life Cycle" page. * Improve docs for related API, with special focus on cross-references and warnings (cherry picked from commit 3246ea514d6da6ff09f411e22b3ba61a7de84a74) Co-authored-by: Richard Hansen Co-authored-by: Petr Viktorin Co-authored-by: Peter Bierma --- .gitattributes | 2 + Doc/c-api/allocation.rst | 132 +++++++++-- Doc/c-api/gcsupport.rst | 57 +++++ Doc/c-api/lifecycle.dot | 156 ++++++++++++ Doc/c-api/lifecycle.dot.css | 21 ++ Doc/c-api/lifecycle.dot.pdf | Bin 0 -> 19328 bytes Doc/c-api/lifecycle.dot.svg | 374 +++++++++++++++++++++++++++++ Doc/c-api/lifecycle.rst | 273 +++++++++++++++++++++ Doc/c-api/memory.rst | 18 ++ Doc/c-api/objimpl.rst | 1 + Doc/c-api/type.rst | 25 +- Doc/c-api/typeobj.rst | 458 +++++++++++++++++++++++++++--------- Doc/glossary.rst | 6 + 13 files changed, 1386 insertions(+), 137 deletions(-) create mode 100644 Doc/c-api/lifecycle.dot create mode 100644 Doc/c-api/lifecycle.dot.css create mode 100644 Doc/c-api/lifecycle.dot.pdf create mode 100644 Doc/c-api/lifecycle.dot.svg create mode 100644 Doc/c-api/lifecycle.rst diff --git a/.gitattributes b/.gitattributes index 2f5a030981fb94..5682b9150a36e2 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.ico binary *.jpg binary *.pck binary +*.pdf binary *.png binary *.psd binary *.tar binary @@ -67,6 +68,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 7cbc99ad145ad4..f8d01a3f29b30e 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -16,7 +16,20 @@ Allocating Objects on the Heap Initialize a newly allocated object *op* with its type and initial reference. Returns the initialized object. Other fields of the object are - not affected. + not initialized. Despite its name, this function is unrelated to the + object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init` + slot). Specifically, this function does **not** call the object's + :meth:`!__init__` method. + + In general, consider this function to be a low-level routine. Use + :c:member:`~PyTypeObject.tp_alloc` where possible. + For implementing :c:member:`!tp_alloc` for your type, prefer + :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`. + + .. note:: + + This function only initializes the object's memory corresponding to the + initial :c:type:`PyObject` structure. It does not zero the rest. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -24,38 +37,107 @@ Allocating Objects on the Heap This does everything :c:func:`PyObject_Init` does, and also initializes the length information for a variable-size object. + .. note:: + + This function only initializes some of the object's memory. It does not + zero the rest. + .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. + Allocates a new Python object using the C structure type *TYPE* and the + Python type object *typeobj* (``PyTypeObject*``) by calling + :c:func:`PyObject_Malloc` to allocate memory and initializing it like + :c:func:`PyObject_Init`. The caller will own the only reference to the + object (i.e. its reference count will be one). + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This macro does not call :c:member:`~PyTypeObject.tp_alloc`, + :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or + :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`). + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead. + + Memory allocated by this macro must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type); - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_New` instead. + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. - - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_NewVar` instead. + Like :c:macro:`PyObject_New` except: + + * It allocates enough memory for the *TYPE* structure plus *size* + (``Py_ssize_t``) fields of the size given by the + :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*. + * The memory is initialized like :c:func:`PyObject_InitVar`. + + This is useful for implementing objects like tuples, which are able to + determine their size at construction time. Embedding the array of fields + into the same allocation decreases the number of allocations, improving the + memory management efficiency. + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar` + instead. + + Memory allocated by this function must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:function:: void PyObject_Del(void *op) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index 3e23605778f05a..f6fa52b36c5ab3 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -57,11 +57,49 @@ rules: Analogous to :c:macro:`PyObject_New` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) Analogous to :c:macro:`PyObject_NewVar` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* @@ -73,6 +111,10 @@ rules: The extra data will be deallocated with the object, but otherwise it is not managed by Python. + Memory allocated by this function must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + .. warning:: The function is marked as unstable because the final mechanism for reserving extra data after an instance is not yet decided. @@ -136,6 +178,21 @@ rules: Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, or related allocation functions; use + :c:func:`PyObject_Free` instead. + + .. seealso:: + + * :c:func:`PyObject_Free` is the non-GC equivalent of this function. + * :c:macro:`PyObject_GC_New` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. c:function:: void PyObject_GC_UnTrack(void *op) diff --git a/Doc/c-api/lifecycle.dot b/Doc/c-api/lifecycle.dot new file mode 100644 index 00000000000000..dca9f87e9e0aca --- /dev/null +++ b/Doc/c-api/lifecycle.dot @@ -0,0 +1,156 @@ +digraph "Life Events" { + graph [ + fontnames="svg" + fontsize=12.0 + id="life_events_graph" + layout="dot" + margin="0,0" + ranksep=0.25 + stylesheet="lifecycle.dot.css" + ] + node [ + fontname="Courier" + fontsize=12.0 + ] + edge [ + fontname="Times-Italic" + fontsize=12.0 + ] + + "start" [fontname="Times-Italic" shape=plain label=< start > style=invis] + { + rank="same" + "tp_new" [href="typeobj.html#c.PyTypeObject.tp_new" target="_top"] + "tp_alloc" [href="typeobj.html#c.PyTypeObject.tp_alloc" target="_top"] + } + "tp_init" [href="typeobj.html#c.PyTypeObject.tp_init" target="_top"] + "reachable" [fontname="Times-Italic" shape=box] + "tp_traverse" [ + href="typeobj.html#c.PyTypeObject.tp_traverse" + ordering="in" + target="_top" + ] + "finalized?" [ + fontname="Times-Italic" + label=finalized?> + ordering="in" + shape=diamond + tooltip="marked as finalized?" + ] + "tp_finalize" [ + href="typeobj.html#c.PyTypeObject.tp_finalize" + ordering="in" + target="_top" + ] + "tp_clear" [href="typeobj.html#c.PyTypeObject.tp_clear" target="_top"] + "uncollectable" [ + fontname="Times-Italic" + label=(leaked)> + shape=box + tooltip="uncollectable (leaked)" + ] + "tp_dealloc" [ + href="typeobj.html#c.PyTypeObject.tp_dealloc" + ordering="in" + target="_top" + ] + "tp_free" [href="typeobj.html#c.PyTypeObject.tp_free" target="_top"] + + "start" -> "tp_new" [ + label=< type call > + ] + "tp_new" -> "tp_alloc" [ + label=< direct call > arrowhead=empty + labeltooltip="tp_new to tp_alloc: direct call" + tooltip="tp_new to tp_alloc: direct call" + ] + "tp_new" -> "tp_init" [tooltip="tp_new to tp_init"] + "tp_init" -> "reachable" [tooltip="tp_init to reachable"] + "reachable" -> "tp_traverse" [ + dir="back" + label=< not in a
cyclic
isolate > + labeltooltip="tp_traverse to reachable: not in a cyclic isolate" + tooltip="tp_traverse to reachable: not in a cyclic isolate" + ] + "reachable" -> "tp_traverse" [ + label=< periodic
cyclic isolate
detection > + labeltooltip="reachable to tp_traverse: periodic cyclic isolate detection" + tooltip="reachable to tp_traverse: periodic cyclic isolate detection" + ] + "reachable" -> "tp_init" [tooltip="reachable to tp_init"] + "reachable" -> "tp_finalize" [ + dir="back" + label=< resurrected
(maybe remove
finalized mark) > + labeltooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + tooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + ] + "tp_traverse" -> "finalized?" [ + label=< cyclic
isolate > + labeltooltip="tp_traverse to finalized?: cyclic isolate" + tooltip="tp_traverse to finalized?: cyclic isolate" + ] + "reachable" -> "finalized?" [ + label=< no refs > + labeltooltip="reachable to finalized?: no refs" + tooltip="reachable to finalized?: no refs" + ] + "finalized?" -> "tp_finalize" [ + label=< no (mark
as finalized) > + labeltooltip="finalized? to tp_finalize: no (mark as finalized)" + tooltip="finalized? to tp_finalize: no (mark as finalized)" + ] + "finalized?" -> "tp_clear" [ + label=< yes > + labeltooltip="finalized? to tp_clear: yes" + tooltip="finalized? to tp_clear: yes" + ] + "tp_finalize" -> "tp_clear" [ + label=< no refs or
cyclic isolate > + labeltooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + tooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + ] + "tp_finalize" -> "tp_dealloc" [ + arrowtail=empty + dir="back" + href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" + style=dashed + label=< recommended
call (see
explanation)> + labeltooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + target="_top" + tooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + ] + "tp_finalize" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_finalize to tp_dealloc: no refs" + tooltip="tp_finalize to tp_dealloc: no refs" + ] + "tp_clear" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_clear to tp_dealloc: no refs" + tooltip="tp_clear to tp_dealloc: no refs" + ] + "tp_clear" -> "uncollectable" [ + label=< cyclic
isolate > + labeltooltip="tp_clear to uncollectable: cyclic isolate" + tooltip="tp_clear to uncollectable: cyclic isolate" + ] + "uncollectable" -> "tp_dealloc" [ + style=invis + tooltip="uncollectable to tp_dealloc" + ] + "reachable" -> "uncollectable" [ + label=< cyclic
isolate
(no GC
support) > + labeltooltip="reachable to uncollectable: cyclic isolate (no GC support)" + tooltip="reachable to uncollectable: cyclic isolate (no GC support)" + ] + "reachable" -> "tp_dealloc" [ + label=< no refs> + labeltooltip="reachable to tp_dealloc: no refs" + ] + "tp_dealloc" -> "tp_free" [ + arrowhead=empty + label=< direct call > + labeltooltip="tp_dealloc to tp_free: direct call" + tooltip="tp_dealloc to tp_free: direct call" + ] +} diff --git a/Doc/c-api/lifecycle.dot.css b/Doc/c-api/lifecycle.dot.css new file mode 100644 index 00000000000000..3abf95b74da6ba --- /dev/null +++ b/Doc/c-api/lifecycle.dot.css @@ -0,0 +1,21 @@ +#life_events_graph { + --svg-fgcolor: currentcolor; + --svg-bgcolor: transparent; +} +#life_events_graph a { + color: inherit; +} +#life_events_graph [stroke="black"] { + stroke: var(--svg-fgcolor); +} +#life_events_graph text, +#life_events_graph [fill="black"] { + fill: var(--svg-fgcolor); +} +#life_events_graph [fill="white"] { + fill: var(--svg-bgcolor); +} +#life_events_graph [fill="none"] { + /* On links, setting fill will make the entire shape clickable */ + fill: var(--svg-bgcolor); +} diff --git a/Doc/c-api/lifecycle.dot.pdf b/Doc/c-api/lifecycle.dot.pdf new file mode 100644 index 0000000000000000000000000000000000000000..ed5b5039c83e2c4a36d77e713c430c8f14901bed GIT binary patch literal 19328 zcmc({1yo$iw(pHYfZ(o;ORz>83GNo$AvEsp8r+@W5+Jxc1b2eFC3ukFmf-Lp*?XUT z?m6$=d&l>U@n}Y~)-36&S*vT+{H=dMr64NK2xQ_yq^duxxJ6`T1+V~Y4J;7(_yEkZ z#x|x-W&l>UCz}!?003YXH@9*!c6j=01$HtPH8!+0GDhU*M|5;@Fa}#Ax@GifeU2s? zKyRB@pRvDT>HCWN?gMzp9|DqRA&pQB!6_lHk1bbLY3}TY4>NgfiAQl(PAft$J8unJ z4XdFyx~2>Jm8d2KxmC2of z6Tf_QFz`KH^d(#BFqS`YCS!u555tR&rL+Q_T8!JtZP4}1U|0WC>XmBJm*M%)%IdM{ zJWWQbULPOi&)^jiL%xM|pSv33qh+qm^YpGyE*>H9fuCnp9EfVu9e#EtpKklOIe1!E zjG7Ike_8LF4Oy-~R~H!CnLoP@@M@16&vs?D^kqWFIy~qe<0QrtGl-NcYFm!Z(-@Am0d1J^DN4er?3R=&8T^e zS@NDu%ForU!X58n85~7sS`8f=u|L!sID_4Fy zxA}3c#(zoU;K(mWJ${6{uVSe*rOKN+;d9RUrk;V2I+NIkGTk+5tc1Eo?8f1xz@Ae8uWSHBQg9#4?RzS-wA){k@ z#q!SHgdUlst0Fg8DN=AE3{@Hxf%7CSs4=1D!iT!tcD?8{tw}F+rZaSgltEzpi^WLD zmv+1aeJ`Pd0VRnT&JV|GIZ_?I3&?VI85o=N(TO~ZSSA=ISP|NTEcgf;4pzbt-6ram zT6q*Qb|v2WFXY+g9k-#kWSlxO#IyTdZJ8wLM-_*YDActV-*R#FQru`L=H0rOT1F`O z7J{eNRx;v}6?;B3Q__U=w4dS;-l;fX6Y#)gQrqTEP*Y8Skd+qQQxyR^F|~MhPP*GX zh&uw$FKKP{Y5y4j{)5vq&Hps@6@dP*~o8Tgi2Pr>BC5>az46k23iu9Y@~308Q632 zD@cQD58zX+VI$Hkh2uqfa5ei0E`RyzYD5CEaCVfYEz7md%sZy?(|RN0B-cKlAtHx` z6ncPY0A={kS%jJW4mb%3L)nPZx2cA6UvX; zi3n&*4&b{!8&DUBe3Kph=ArfRWV*B^ovXtOFMFlExs7A6^s02c-*mix$M;$^+vtZ> z@VYtGNMS$T?ZNi#`ZO~?_Yww)2^qiXHa%5@j<2#Nbkb1&gA<&a_6GxI2pTDIm2C8E z2SK3dHV1H%9D;Bv-0V{fFg)#D(=Jg4>cZD+=|a0AMIs$=)@ok)x#k3%ecR;)s^n6w z6S6Xg%><*OrcmLlP{E{-2-YHKSy0JTm<;a>i>whzdx`>#wULjSEGSAsr66oY#-S+q z-AF);J4ya!BBT0QJU7*O_KCFzaMBI}&fXMbYYSmCM-|e(r6qdLolp_IGl?d8Yy#V) z#KKBe&4u*EuFgKQn}5R|3@}!(A36okggW22?_O5p9m$8gF}TIC{jiK38b$MSPJ(%l z1l&%sFk&51Tr=Ak7~Dpht*%dVGpDPZU9biW!%5xuGCPndm3V_F%_I$c!iyYz_#L^F zI@)&}@u9ApYxFtFc(nuyf$CaW*he@W zBF7qE6+^@RJ9vk{*HZq+(1WHbNPfsNtmor({Q4cp z00AuJTc|UoWqqi4?o%(Xx~u{~vMO?DKFn1Ca|eo%HCsqmacyf9 zddyCIUPFz#v$=?`$O;i6UzWAa4&C*5B6GiJLTqqir(J5Kr4DEiqRQ$SBm}mqh`jEyvDB%BG1{fI-7iY+Z799Cq^Q4wP-2S_w7{)> z5vgv|53xa&>a=9BIVg50Kr`@eOH23TjNZ*iYb>Tm>~_}I6^`xHq`pfw;vBJNb@K!C zCSOe*+U7*_aT|fg%+V2e-i}1kB%~>$DG8mS19dvXOzD+OYbH@VdqG8w-o`q~;PV9r zb*KZxlR0V#UdYqI#gB0mV_Ts?*2KN;7`#ofShtpwIEQ+T-+_%LFlPRIQwO^55??vR zGVBIxI&n)ScvD!jbZg1dTagY|I9d{g&>5KnPkFFu!bjd#=sptFS~nu%f$2SAW26Z> z$b~d+~Z9f;Kpt9D)--wsnkQjY$m# zv=m$<4tAk$IJNGOiMQqKw*4R9YEyEE{ap7~sf+cj$xzxc@*HX)?os;QMALr;5cjc) zFpgqNPKFG|nc$=!eN;NW(0I=*o`}UP(Vo%`BF&jyqkeICz!(7}6avrNo@%hXoulI4_I<+Wq#W|0A zq1`gWbwxk&egbj0ch?bnnoK16yH~BZSs>S)VTTQxamFi}dojJ-o-em*``Ead%sucq+IysTa%?fOM;f>{VidOmazQ}2_Li}j2TSD=(Oq^i-?&5 zl_?rm1jAZ=hC77|z_FMONVvoW^Gz?BKJ#znXF!wkvjo+nX3MQxQPc{$?)6+Q4Vg@B z4^vYQT1VCC7{nRYfIw)aBHwYn-f5t3e}9E?^I@1Q5;j?oTnlarxiwuZbSoyzUGuo6 zmPd!)Bi)N;crW6WnpTKb08zEIX#zV%ORvctbkvRjqFHU**p%PR;Ms~_-CE{e09A^P z59A`$&G*^**@G1=123U5*0$-9JS>wjvuOsl4{lQyYjt7?jTqR;WGWkFfa_(-t|JZM zv%5?ucmxpZRBihwh^VKl%2JfG;zx5k%bCC@M1a=@A*fI4Rk?EO4vL3rw^GH4r=|GG zt)Axy$jlrU4kh1dM@cK`>o^R`N8S5=(vy z#@N_dK+idT*@1tBZNK6)F*hd(Wv8bY4Zz3uY1|Wc{m=e}aKR8>E{we0Xb zbR6Rkas0`j(Q`S#J<;`sA4XAVhQdEQEfo|-5bQ6CW1gCd0q$ADMN1MhQ2CZ29*!>{-jdQa`VC+~TwE?Q2Pl zKovM>#0a@D#H?4>>Ua>rTfNRs)_r#;L29q-wxUng%M7^{*u8a$!;E(H=#QpN620Vo z3|)%E+}609KWjg-mP-|Zn49!woJO<@XrzM9iI)yTGlWmOs-|)`Xy|$vm#)Ah?7jy* ze1?nea5+HBmCKjsHfcDV`X=L@1M^VD7Amu5){Ri99~ws_ng}hrBzh%w#6qcAxt}Nr zfZSPMB8(@kDDFKHd6cBZ zsI~$_#eZU)(onPtshQLXVJe!ZS4(lG$$^qTE|jBeIW4WP)b)hX(NRcxxxQIiKN97j zqVg-TDb#j`iOe6{ElM$(%{)*G68@CQtkZP%?K+{w4W77PZ^N3y`(+ICO3h|dth&fR z-aX?TlYO*?FcyY*d@DE-BwSJ!O>Y=cA5G%lU^i=%g%Mg3^m$$SjjMq{WkXs;MMaj) zR&K$3o_ML$*eN^RWCcA`NdbjnZXRVJ1%S>TOSj(FS4X$&wDR!`3?^=?Yd2UfklUx9 zQ@1)~wi+tfS+UsWiOo!yG-E(-Srv3L&qw z4!JIp?ja3!AM#OOYkohET4>&gnhz>yHy55he{<-WXWN$2$j1E0w}&bDIjXXkXl6&W zm9-5>2~nhDUXlcrrcg=h&D=E9(O1bt6v)zt1&AwSX<=0rsc5{dvY7uVD}?zeG2-f{ zPF-OU-BQ}kB*9_blo_rfw!c2X2CbGUnxjVt&y|V1<*9cF&dL5AgR?G=<17t-TaG~^ zIYS2XAukWL@oE*AJ?KUm>BpG>(c*)LVOucS#uU>wI+AKzstF{_A|>>;iKbC*7*G+B z9q%QBf{}Qh5CTk>zb&<7LfFah+UW*vf0M8YX7uXzVL z+c{|a{XWhv0?SY-U99wGNYZ7cDjJ8ZFh=7j9CFhgirY!X6kzHimHNNn|OV_ zsN$ZZhDVA+3ri3miaTQg;}J4Pnwp=fxM#JhG{saA2sb`CZc-w&PJ^spWJkNf7)Cj?Sf>(tz z7A=`-&;%eS6inaJGUd$Dy4Qf?sT)iB5+k6z#^|$MSiIw%!q>~5HazwclCqE%50Cn) zNhk@quC~TPO1B2SxyXZ+ zosprX>gL~;)FjYH1gCzs8HQ+a^ok{+yg$v+k^R6mpUKB+G6B(uP-1`^UczZxQL@t7 zeA7MqYE`Gn0k$3FS&&y-yE(Gx@$ll2LSX#Oqy~W$g;~ZJYSDavM}Sl-SFh6d`I6Y7 zwYhin3MI0P3b-4#jcDc$p;L~a(*lPTN5y6@V3#8s!E$IN+fm11;BM;A4Xc%5-r|sO zSDWbE>#uSN8Iy?%+dSCT3acE5!vS`pwIb#850E1MonY#R@pI^eJ@SE(3 zJC?3}#ZY1FSuvfNeCk-3w2zcUdK>u(Sd}~xBe}JSnkBE3e~O%7zJ&|lGMqO$v=NRL z(8xO7rPUzjx~RqUF6?a`Q#Dp>{Bf%cxuhJ)M;S-8LiWyy0?|)JUkg%l0>c3%ZVDD) zEo&gjE;~Y=3U>jUY4r)?Hcb4Qdm2!SjKOYFO%Uow&_x?Adx8Mf-{2cBg>>K_z@8w_ zKS=t3bD9d>Qpxp!k$4vg!+fX22uUo#h{3&pfW_GDbyB_tXtX?Di7d4;RQZRk9=T#vMOHUJk~ zzN}M(-B=btPN}-^R{i%&MqR6dva)BjslGnxyV5gvjFS~&9O89Y^7Xu8bvW~B#v31g z-hKI=&^l76OReWC<1n-AZtEjw%U3r)a?!Tfu{xkwNzNoLFn${B=hBY{eR?4!5n?Ms z0)*0ws|ih{v%GqVX6X9n@=Z(G@ev+HezV>VEuP^>pm~3Dtf3B+Ow=oIl;YSAS6+^u z_~P4Fp@*-qhdk4R&f0f+`*=OaXOE!9%N>`Z_veo|UW9gzddg+i2em`oM42G2Si_BN zi!xGvvemUHN@MS=C)ip`nn+g(Eb8>ie$z6ebG6+=^`*&LL`=-sSga4u443+TK{xYt zGC8r*`{=CUQPZ(3-sMSdyHdz|l?F7$lrUo$Ex2Ckp~07Fa=4QNyO)z`bulnDw zNjo39NGvzyEOVb~q{yA0cz*A6^}Wk-*uNFiS~BBWmL>uYoHRUPOjpCFN~$G>sF%eBWQu;JZ>0f>okG8>Eo=EH&q6rDPaN&7nl5%eejBHmrdk{60m(H4X{1 zV{ip&%%!DPS(y^s)@{X^Fxtz3YoOhhHGi=aFU;?68G1VnRjAzenFX>VF_djkt%g_v zMc4>I6Yg`7H$EMB_MUk$$d;irSS#bLL;&@RP07~}3#Fo#m4LKiKtQB8@%jOI@dwXI z8v|OZR~qnSC^h2~zIik~y3%=TpXtpmR#G>4c*co&?|eg<`qnHcYYFbvTD-VyMuyNu z7F)$}<`75>^LE4dA7l8lVu<;@)uM+bpxM~q3PwkUJNpla$}BKA4LwbQALOQ&2Kla% z-_P$3F*{f*af%h%C?$RD->YV3xbS`~rIA=MlxdMAkNPz1>t6nP%f#JVgw&X^UnLh9 zGVwSpG5oTRPv`cflXuTccuRb<$M4mf(O6*mftg%)$^wxTX}CS?6ap4&BC}PBhUt6_ z9Q{2!Gv10Hq$##=1!^jM@Y7nd$rC5#l8(BdF9U&9T`2@7B?0+$_~nk^r6_7cQp}n( z^U|M>h|AI~c7*uiM-u&*vB!Jt{6>z&{qY+_|S#c)qOcSh| z1)tnz@CMoIQJ3%bfxrtF^|5aqBcy|>TMXVy8)@0hfGfmiHOl4qAdAlY;KDdYa{Fcn5N(Uzl31 zpO0r8IHF{$rZ1YxU)5oaeH4aABrdi~>nVO?KfmD2NBj-D4E?lDXqh(cqr z?v@`yymLYpVwi82*aJ_rvSVkIPRcv7juyg+a%^wbcdbQiHvz?X%vpyOYpp1 zYb0`Ubn8KB0kyVI59z%L%v8G?MR|X3!g8RnUaxMrXqkUv!El$>i^W|#6k^*2Yt;(km3HC zj373@wqip2ttwCiGUTogogur&Ah)Uyy((0MB*YWMkYQ_rR357h;Yu0=OUq!_>BaUz zVh5zp6w(5^BWF*!Cr9758{B3|@5#((iRd9lN%_3)LNIKz!;*(zfEsMBDP*-|Hkf8) zswu?6!4@jF#R8j82g`z=|9LbP9|nIR7DzyyUuqRkOnnVgIZs*ermorvd&(P1$UJv` zhM%?Tc$o(@0FXk6QeXH12tzC6sFhAJ|o(IBVGt~WDA@*Q9nws3JM*9oit=5 zK6-b*Y{(>$cpuLKEz0Lst@`I6#weM||S;!~9?I$1kYy2mWAZWn=pbfIL6df42YaNk1F=e*_>; zVCSzV{kn1EHUTWi{m1Tzq%9(1A6^w<2&a8Q3b~(LAOXk`;g|=~PC(msfGT**;ZK;~ z@@rS?X58+qv|xu{g7zWRtH#nphrOZ**8TI&bIUqGZ7cHEl{DC*aq%J6n)Z8bnPOjr zPve-9Ic%ytSa zO=|w75xUTZ{t-G#epb@Yt5qFjZANXBb0cXf`!mgBe%FFt2KL1TMDMz%f99N!O_n?)Y{7@l9qMuw(6JyZM=1ii&d1w{BMTj+0B0p z3ELA#`h%9SJ;Sy?N8~pxdxmnN#*T&#=5|iD4!?QWlSv+I{e;pqG?m5G#b1dUTY%M^ zmBBWSa<(?Mf8`MeTbo{bZ0iQl1~77PfB=ka zTmfe1hz6a6tXe3G6t~xvIsdE8b2XJZdTB*)8}SBpEI(u{RVd; zU^_`;b5k>?U)B7|qwHjCtp))7F8_QY`RB=VN1wzc#;kx}U{2}p5b@u;<#&b722Q`Q zqKbpF@w3Z*S%txl#=mgpe{#_?ViYk0JN)ScS@0j{K-PbFIS!ok7?dIp{Ux0Aow|NY@}AO4o| zFQ*;1xaOT|R zTTF}$X@q5M&9|zz=EK>{7`gq@4QjJxC0`x$oiTpC?Y0$HQ%~d_wdLdULz4rJ8LMrI z+6AuS5!rUz>o{9)4*iIsd)o!9po!veju?L7-X#<|_ipoj{no4fKI(O4qT_+;oQLZ0 zo2^o;OS=QAKt=#m@i%J&DqjC7`WKVthIIva*0TAMBn$!TsQl~JLu$;*O%4M#8WPf@{qmA)%JWAZzD*OO}XN?dfl z!F3!*3rD7G$U+l`1^Pa~AY!`IPdq`grQRc-d5hVjJ!? zM^+=aO#Hq<{9eKB0J3`aemTTfPu6;7t$!0h<<;P(?i7p0iZPie5^iMjF?}TXAC+GvGLf62`8(7adCac*RaGn#+9 zX^8ucfg{@2XgMPT`lyh2+QXdZ@$9ek|mRC^&z``C_m*7#mQF{9I*DR$+!jB8t~Gt5gS?q}O&>NP<7 z>Tntjrt%9_ z3-a2$gud+Ew;Az#I;am~NWn5fHB38v9$KRW@x;|V$dHh9mPe|Q)NSv8by4!~rl65>4Yd>bCj18bVXw0W6MwhenP+J?m z&GNWb*^Y}fnZ7&=vhSo%Kor@y`F2NQ1SF=my%cS4>}OjkE9WIY#Z{9|OPOUHFia-9 zZmYFSQj09j0F_?W$k?fFMMZo*<_oF-t`Ua7jeiRP+(#F&Pd$FCl3oJv8b%yMB01DS zkC~k)wH2%y-j4TDip)PqxbjI0T=_M&^Y<=8ev_Y5z6{Vfp?K~3qn5hVJcd+>%|%rv z#2%ldw^U$iDrMDKYffoS`Df2;XwK<%?+2z}NH!!wc9EWU+rlR+GIKJ zQA*^*I<={6#wx~=9WTA8vSBPRBa<`9UPrp3pLKxocq2yya^Y+o(T}^7&peQI2*V!k zU=MBH=R6){A@=m2&nTE11e=ZrRo5_&M@D%gW;lZmt*h2vW3yN$@6;zH?`cdVwgaOJTZp$`~}h`>+#JRnxSvyqXu=h zeZ?Xq)v&tCj*4sw8EN-e_awKydrk2RYuHq7@sO|Y8NaHo&X98TBG$E`aT5&sL4&=i zX&-5WC}z9_H@L2E2O?VU=?=294nTY{OnkRlZTf3GV4FxFd5}m^c0Nk%5W7OA-aZTbg_5 zx|Lp~7;Sc@XgH@qmUOuPOU`+%)h4tGl9lFOf1^peZ;#o=C4VFP;&>Y_N2~pzA+gj} z@sFUOJnNv5g!sEJwV7`Oc-6=6H_@*IWO2o+#@h zcDn3p=J^L5hhbTOG%!|8z9A7@ z0|O@sk4cAy^Vo{Bk9t)@3=b0Sgc;kUaFuD=G+PGy=1+02(A&p?JY_|0OGLOi0(+7E0yB%3YcNCx1LSvj_FU&jGqc;bHZR{^!LE<`aG%&9 zP6i>EXqzD*^nT>EA6*~xQM!Z-?|Hj{i_^U8!IPad^`kSaS(yE z@L2uH+Dn)pC(E5cR!wsG4?aK~Ao7CFl5?t~581p=9M{(}rnT(7?*sLo*OndMXxthp zVtj;l>J%D!$by|Sw@sTuI=bIPlE}@sc&=u0Mzu@gPKjOM8rEKedl~mydfLUl4*GqC zmU~d$s{o69o`XnqW5rC8{+N3`mF`yy1#e8;v;`|@%z0A!4LJRdXFWlxm_KF zqoaKHtBulk=P|T}mCe`W9x3n+=@BIRF=>L|M)+exg~=Ke-8=;SKl8}CnM^o*im%A` z3>@xe6lo=!W$8arlC^r9o#B|^!jqY_e!J~=$>WvHaq&<{8jjbvE94PM(m%zoz+~ti zabvY6vEL)vySjWF62m@$LTVOr&(%)sm1yRTxDj)=7d8@!=qnORI!VrKLB2r?p@|_` zlrgQJgd}8sC!Jzn5^!*#j1ltGyT^ihs=Fi{ik1?3# zI-UJ>Xjd{9+JkR3PM2w#+A#(WQwl#HAB)6P5(ScgVbMR}dG@t4FNNXuhm&7isIRKq zCfI*Wa2oWU6Irrbc5;9ohWYAE|DBr4{1vxwWr9gfCPP9@il`EHOLR@sAi2x3`ceEf zzmfY`Uxi_!1r5_oEupKxl)xLzT_@TAX)S*O#5(9RdlzW?^;Hq$+(mUF{`(tev_R@! z(uw2?g!3SnNm*VriT>TLd)`4%Vj~9C=m7oTS6_6`BW?>XRcRMzWgm`J0DN8~T3iJ(y>-7pYHh(5r``frsWKw z(4K!hZsvHW8_IUnG5RiO@1P^rOdxytYr8LpW!1fl^l{1ubDu^#XUnBR)1l`Q54d#XhTUq{aAZF=g4@{`(h@_LzbRISjo>h&hg2nu^ft#WsCBg% z(XegeOC(5CFVpGuar8)~)V#dq`Yby+cbN}*{}totNBO|z7eLwwE?2BM+VW*qw^Tg# z9qoW;7A7+z@}HN^e<#aI}4Cm!OY>5y~WJ!R0{*j ze($}(C-X^>y=e=$)}o&iFVib)szkgkiP?J0ztRZ9x|1SbJT{aPIXHV8YydHKi$3|P zZo&4EB4n7S$q5*(>(EWr;s!)nLH6fjN?9b7Z_*Dw&p+{CbAlIChj7YD&?tl*lUl3q zd_^?oaD^Q*O(bs*`kp!>zXCsQbVU_{IeNkxlI*|kCIR4O5|RMi1;9pgulLb{4E%7j zTx)Pae8+)1Oa64F{CWNSD5W z7s17ouKEqSYq9>kyS=F{+m}UH-Z}UxVyy)*Zlj^?E#5*yeT4O9m9q@pmnB$j?Ow)o zIV%}DtLlh!#fnktg*wGyPO&a(+f{eZ>?TDdLNt#6%q+>EIfW}t!ZOr%IE4xIb{gt= z;C*s^YeyDsOTTS>jp>gjmQE8NgJ-R)q>)~!R&CfrYul5Oyyf?QZS!u6;sd&XMWnwm z5y|F+0%Vy z^UFRVm1y^tNMn}L2hq1^nENB@sMwQc$ZM2^h!G8OCo;;K7!%=8_ducK6K&&mwwk02 zgQKCPpP8%+?#Kp9&x>g)w_d0|F$=Y?$YK_>BlU0Ig+S|5Y=RMT0<+WLL|08Vgb zp@TVt!5)r)7!W0PYyxkGFQh095|kTjlCfwZH2?;`#l|sJBcps*BbG_-v`_?=qA-|0 zjO0SeJYIji7O{{9$v6v2*JR}fiVV2}xuyH!f5d(zr@rRZR3eO&PFyI0k`$%FppvYM zK+V}jkI{0 z$dg(o2=G1bK!UE^dbDM7@nXOblQ;o5kRW)Vt1485(+W;>tpdFmZHx?As0S2iA;>Hx zdkiO4Xc!L@uRHS{05c!th&tl z+A+Ty()~;4IH$RPL*bsuz#kNjorRtAFAVp0(b}IB;U5~c|85JEZv3!K4**T<(hGLj z34O7jjxyf3Sei@kBInaR1hR-laX$ht(xW#%wRAqy0XOm`N>3ihbv%<)U9WXu@uJFs z%YzV{S~&?py2k3E_QFP@c18T!mYP^2Zsfk9jKz6nw9MS?G>-a-jCx%f?Ld;??XtOV7z|7W0HW{W4STeDl*`!r`0KX4p0Im{%5` zM{trpyc9iT?o6w(a^KV$?)>8GxFMO;m1;|@Nui*XL{}wF%vuLC)`*Vk>4aF_{~+_uWA#v4^?t`}z9B#zE-QcLZ_- znN8HTQF?Sz6pA9AicPpeuMrM};>2^r6~>`F`HP;AYXJ$$LIAlfqT5)H{~MV4-}vZ%3sip%(fr$&k6!Q3qYS=T$~(#R``d38^i>9&hQMQ|F98bOZyOsNCGbT4Dh>HWj#`TmJ$OQyFPe`2X>`W}|tlXSW zT?BGJwS}GaskhH`m4k_!gNx+}oBz$u%?@IE&hb?AZw3F3U;oh#763aJh>3-ji;bNf zzyV}q0)p6{8^z87WCC$;us?m`-|FDtWM|@L|J4^300$=<6E`P2=(*rO>>R9|e-`{# zX8YWiztes8f1|@?|DWq{S=qS%nc0r2SSzT^AEaN_#-lOhkGUHLW$39F1HL1~$-fpzA)1?ctR#y0B*7 zt(Uf~=lD(+vkg*?Prv5<;j+Zi!^x7iZLLq(B`+~=BYtz^+zf&rO2aVk!n$OwNzLnS z7kP>Chn|Lm@8gRm$l$}3HbcVf4#V4dFY2FGykB2w&zfa;cTcytnjZU6Bz08Z=cCn$ z73Gd1f9Cu&a|NKadk2K@>)Cg?VPJU zyRNc8>uH4v%G<}A2-F$@PXYVn4hG^)-Evp{G`7pE_yZkIfyWE6&B<{2NCBnQmIiP8 zS%7O@nD+=5IlswuU_D2!@&WolAHqJW*xM0bWv#sKEKjV_dr{FuzA@#z+{6>Ck}K}b zEAF4M=U)~`(7za-(#iH4;U(;m<>bcI_iOZphUyn5rerKum&Ek3+KWVrJEvihjcGIU z8yABwW({<{D(hh-zi%N2!1eNf2FFz_>@qoi$NKW!M)rY{bAU&}JJTuI)fz=?%no+Ko(adBZTh^gG)so@ zrxJav+h6)+)lb#(?oK6u{-P!+;#_@uO|wa&eHgfCgrFt(4pu8E+;Z}FFC(j>y`c>< z&*)sy)0#}kUZ|o6k(g@ym?H$=7n_Vrq}PNaK8{|1Hu=Dlcnzedq6LM7>03HWSOxB2 zw{r0JQJ<4+f7Eb9LW1u+ag`NhjE%Z=YEvz01FEFV;sGH&2f-jp?n`rIC5MV3pc z`eLWf?UqN}DtBm}jh~m^RK7a2EWdOQ-{sBi#kk_C7?8#7ez>jNdwgKK)dC|lUd`uA zw*!|z-WS!nc=w#9H^ry9g2;Un8Nx9>BCKRLREYSB0XO%ka$alwnP0e_20Dp#SH`-S z51X9Xl5OytQX|4}?#SPr!bDQ7^*cw|T-oW0^oC?Vb#;S=dWGvWE39<=y^}NI`unTu zFjwfSxG-0uCe}1bb@oW=4Qo;#v$b+$ouBKYT*w~Ny@YUjsNB2G%;G3t%z7h8+1EH| z+TTXX(6H{thpC5TURdx1XEuSpeTdLQwqJ>b7WvfL8HCHOgD?~eO{VKJ9&yh6e##xQ z68Y12Y-YIeMe~WMuF%~5)3dLdgqgLQI-=+vZG4tJi!aT@EmrV>!MU~ zDugB94(u9Z-WO^W`6Ay7x$k{Umakt+WO~C|nV^>eI?Bbw2cf5CO%!g~<3`)>8^Y<) z_eIp6ETctJ1I+NS*iVHU6Z9l|&LQDnAB%MQ?cBuga&z6ppI9MC=zy$xaso(r*nAQAT=vn3l5rys3YtElgV$zNYp#+@0Mzg2?)=DL?L%W6J%h1*7;&*wKdrY0+dR z-e%(+EEngz2Tm9<<>2Nl-G)N?X%dQn?`RULDzqxQ(qPIs6$DnL ze~^`(fvGa;rlM@A^W^IlbqOrgn~EEdURh++bw{P`!IaqNYQ+@p!R($+V!Z_UO$hkW?Nj_DvidIDY1{S@P*yliou_{G-#PEO`+LaHvXoyJc^j3n9 zsEQwQ1e332U#|+TbSn*&_6t)tRADuHeM{3&cv1QKeJkVOXZeB*Y9mKY7uFhQ?$vOH zH}uB!oeAH`-_Y}6vL>V(e~QIV#OE~HWqnO}9?!xB?>HrGE{{2d$5D<|Ol{&gy-01s z9hYmKI0ZX;NG%soAk#?rIU|`54)<;4a9{$e>C5;OUnK;TXl~gB{LU=I7A8zN%Br{z z%t!uFj#iMpgjM~`gC%5Rut?W~7!dZv*8qH396upd!bEBWOleVI^h;XU$3~gE$`^O@ zeUEdG5H{u%zk-Z^-qQ2yX7|Se7Rbr{*9H~#f1}TMy3hWn4J!Yn&xjqj=y}#>_#hI0 z5wV|kM2I|!?|GQ8LS0*MFflXA1BJBS8*w?Y_%Tv?3!H1-EOq?EV%>cc?b3}CC8U1@(C{8>Cih81UVBCzDO zjMIcim^@jt?|CLc+dWWcR!%V{KHTFNx5Depe5hh%qtGUofDQ+0-BFXSKzQ~XNjc7L z@L$7#U!M8Hi>$1FiCozKy>{RqkM;jr-v5?VsAwpNi2i>H1K9tn@lQp?|NAK5*Ba}e z7F~aTQ2vkT=%1qi5h>BS0 zXZvrd%Kxt?`UC@Njs=1|g{hF=^9@i4 zX*415Kk5tX?}IQAv2guf1eEvH6VZ~LkU(l%ruJ831)CGS%1?#Ke-920a9GI7?D}>C z#s{;?*=~kIc7BuNf7^8o@asY5XP**lX0kv#SgiHyJI|%v-#(kYDX{t2=zc0uIy1{t zT4;_<$Q$pYS$J-qip#;s6e%NoGNx(QPqUlL8unr+_qauPf)-hhAji8-b7ap&DESF~o9wqN<{{8p5~Uq&UZ7xS52kK$JFy6mka1Pv z`);6U5(4+}XJ3r;#|!EY+B%9&W?8hQtuvl5*lAabY3$v3SBjK*_)uFnX7A3(D^W)! zj?e7*(U-12bmGNPo6z zKF}WtUQL|4@y%F(a3v^M)VR_RT7i5xDR124n9h#j;SDd1k{hJZNHXnN5&42U zH-Hf0uJ!ZF_8y8M9AG)h%fVyFKn`jRDQyyDdaaU(d=+ZYX_e^DCl)v*=~niQ_y7lqeB$D7kiArh9RuE(lmxd zsgUa3?ajH5cMSievj@?Vm=fu*+3I`!8>Xay_ENG7UBsaFm>@&L&^}|s??B`mpU#ah zKLjG*N+Uh=k+0N_ZI1G$`rF`Mt*!}J+C`mmzG|tnm=2-r6cM`Iqp>w}{X(RgHet3| z{4$>@{1e2UK_OB)lR?8NYx&SMPXu;`k-6S`0EXWx5Y$|{zqR+>XsW1D^;7F`t*Vk* zV`1UpWKlRB)vU3W1gP1Ip4*0uYGA_zgeqWvPO(#HxVl>eE8sMc`w&@YzpIuvSd%<|{rgyl&t_=i-Hi7Jq zp=50GYp(kPnF2X}W2%40OP{B4WpfW>00-NzB2N?d(;i{~>+k7M$=3F1^Y8P-_j^uz zzKsCbe-qfhFL9{Gxlx3`ctPvOTyN(2*o8x>@Gvrqn?0+LkUlg(z3+u86B5gtms)ri z{-Jq{a5SNjm)isfhRiZW2zzP`08MrRK-LTuM9GZ}!NUTeh)D{*h#%uk;~sutJFcZW zyAswVBKww{H>4%Sy13D7WsO1$Nh4xR)2Sm@9qS%ir`L{7{MAC}>5B@3vpZ4-5kz_g p$M!7V4%?s5Zqr0H|LRLeC$NK)o5PcrfFN#e5F! + + + + + + +Life Events + + + + +tp_new + + +tp_new + + + + + +start->tp_new + + + + + + +    type call   + + + + + +tp_alloc + + +tp_alloc + + + + + +tp_new->tp_alloc + + + + + + +  direct call   + + + + + +tp_init + + +tp_init + + + + + +tp_new->tp_init + + + + + + + + +reachable + +reachable + + + +tp_init->reachable + + + + + + + + +reachable->tp_init + + + + + + + + +tp_traverse + + +tp_traverse + + + + + +reachable->tp_traverse + + + + + + +  not in a   +  cyclic   +  isolate   + + + + + +reachable->tp_traverse + + + + + + +  periodic   +  cyclic isolate    +  detection   + + + + + +finalized? + + +marked as +finalized? + + + + + +reachable->finalized? + + + + + + +  no refs   + + + + + +tp_finalize + + +tp_finalize + + + + + +reachable->tp_finalize + + + + + + +  resurrected   +  (maybe remove   +  finalized mark)   + + + + + +uncollectable + + +uncollectable +(leaked) + + + + + +reachable->uncollectable + + + + + + +  cyclic   +  isolate   +  (no GC   +  support)   + + + + + +tp_dealloc + + +tp_dealloc + + + + + +reachable->tp_dealloc + + + +  no refs + + + + + +tp_traverse->finalized? + + + + + + +  cyclic   +  isolate   + + + + + +finalized?->tp_finalize + + + + + + +  no (mark   +  as finalized)   + + + + + +tp_clear + + +tp_clear + + + + + +finalized?->tp_clear + + + + + + +  yes   + + + + + +tp_finalize->tp_clear + + + + + + +  no refs or    +  cyclic isolate   + + + + + +tp_finalize->tp_dealloc + + + + + + +  recommended +  call (see +  explanation) + + + + + +tp_finalize->tp_dealloc + + + + + + +   no refs   + + + + + +tp_clear->uncollectable + + + + + + +  cyclic   +  isolate   + + + + + +tp_clear->tp_dealloc + + + + + + +  no refs   + + + + + + +tp_free + + +tp_free + + + + + +tp_dealloc->tp_free + + + + + + +    direct call   + + + + + diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst new file mode 100644 index 00000000000000..0e2ffc096caba7 --- /dev/null +++ b/Doc/c-api/lifecycle.rst @@ -0,0 +1,273 @@ +.. highlight:: c + +.. _life-cycle: + +Object Life Cycle +================= + +This section explains how a type's slots relate to each other throughout the +life of an object. It is not intended to be a complete canonical reference for +the slots; instead, refer to the slot-specific documentation in +:ref:`type-structs` for details about a particular slot. + + +Life Events +----------- + +The figure below illustrates the order of events that can occur throughout an +object's life. An arrow from *A* to *B* indicates that event *B* can occur +after event *A* has occurred, with the arrow's label indicating the condition +that must be true for *B* to occur after *A*. + +.. only:: html and not epub + + .. raw:: html + + + + .. raw:: html + :file: lifecycle.dot.svg + + .. raw:: html + + + +.. only:: epub or not (html or latex) + + .. image:: lifecycle.dot.svg + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail + below. + +.. only:: latex + + .. image:: lifecycle.dot.pdf + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail + below. + +.. container:: + :name: life-events-graph-description + + Explanation: + + * When a new object is constructed by calling its type: + + #. :c:member:`~PyTypeObject.tp_new` is called to create a new object. + #. :c:member:`~PyTypeObject.tp_alloc` is directly called by + :c:member:`~PyTypeObject.tp_new` to allocate the memory for the new + object. + #. :c:member:`~PyTypeObject.tp_init` initializes the newly created object. + :c:member:`!tp_init` can be called again to re-initialize an object, if + desired. The :c:member:`!tp_init` call can also be skipped entirely, + for example by Python code calling :py:meth:`~object.__new__`. + + * After :c:member:`!tp_init` completes, the object is ready to use. + * Some time after the last reference to an object is removed: + + #. If an object is not marked as *finalized*, it might be finalized by + marking it as *finalized* and calling its + :c:member:`~PyTypeObject.tp_finalize` function. Python does + *not* finalize an object when the last reference to it is deleted; use + :c:func:`PyObject_CallFinalizerFromDealloc` to ensure that + :c:member:`~PyTypeObject.tp_finalize` is always called. + #. If the object is marked as finalized, + :c:member:`~PyTypeObject.tp_clear` might be called by the garbage collector + to clear references held by the object. It is *not* called when the + object's reference count reaches zero. + #. :c:member:`~PyTypeObject.tp_dealloc` is called to destroy the object. + To avoid code duplication, :c:member:`~PyTypeObject.tp_dealloc` typically + calls into :c:member:`~PyTypeObject.tp_clear` to free up the object's + references. + #. When :c:member:`~PyTypeObject.tp_dealloc` finishes object destruction, + it directly calls :c:member:`~PyTypeObject.tp_free` (usually set to + :c:func:`PyObject_Free` or :c:func:`PyObject_GC_Del` automatically as + appropriate for the type) to deallocate the memory. + + * The :c:member:`~PyTypeObject.tp_finalize` function is permitted to add a + reference to the object if desired. If it does, the object is + *resurrected*, preventing its pending destruction. (Only + :c:member:`!tp_finalize` is allowed to resurrect an object; + :c:member:`~PyTypeObject.tp_clear` and + :c:member:`~PyTypeObject.tp_dealloc` cannot without calling into + :c:member:`!tp_finalize`.) Resurrecting an object may + or may not cause the object's *finalized* mark to be removed. Currently, + Python does not remove the *finalized* mark from a resurrected object if + it supports garbage collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set) but does remove the mark if the object does not support + garbage collection; either or both of these behaviors may change in the + future. + * :c:member:`~PyTypeObject.tp_dealloc` can optionally call + :c:member:`~PyTypeObject.tp_finalize` via + :c:func:`PyObject_CallFinalizerFromDealloc` if it wishes to reuse that + code to help with object destruction. This is recommended because it + guarantees that :c:member:`!tp_finalize` is always called before + destruction. See the :c:member:`~PyTypeObject.tp_dealloc` documentation + for example code. + * If the object is a member of a :term:`cyclic isolate` and either + :c:member:`~PyTypeObject.tp_clear` fails to break the reference cycle or + the cyclic isolate is not detected (perhaps :func:`gc.disable` was called, + or the :c:macro:`Py_TPFLAGS_HAVE_GC` flag was erroneously omitted in one + of the involved types), the objects remain indefinitely uncollectable + (they "leak"). See :data:`gc.garbage`. + + If the object is marked as supporting garbage collection (the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`), the following events are also possible: + + * The garbage collector occasionally calls + :c:member:`~PyTypeObject.tp_traverse` to identify :term:`cyclic isolates + `. + * When the garbage collector discovers a :term:`cyclic isolate`, it + finalizes one of the objects in the group by marking it as *finalized* and + calling its :c:member:`~PyTypeObject.tp_finalize` function, if it has one. + This repeats until the cyclic isolate doesn't exist or all of the objects + have been finalized. + * :c:member:`~PyTypeObject.tp_finalize` is permitted to resurrect the object + by adding a reference from outside the :term:`cyclic isolate`. The new + reference causes the group of objects to no longer form a cyclic isolate + (the reference cycle may still exist, but if it does the objects are no + longer isolated). + * When the garbage collector discovers a :term:`cyclic isolate` and all of + the objects in the group have already been marked as *finalized*, the + garbage collector clears one or more of the uncleared objects in the group + (possibly concurrently) by calling each's + :c:member:`~PyTypeObject.tp_clear` function. This repeats as long as the + cyclic isolate still exists and not all of the objects have been cleared. + + +Cyclic Isolate Destruction +-------------------------- + +Listed below are the stages of life of a hypothetical :term:`cyclic isolate` +that continues to exist after each member object is finalized or cleared. It +is a memory leak if a cyclic isolate progresses through all of these stages; it should +vanish once all objects are cleared, if not sooner. A cyclic isolate can +vanish either because the reference cycle is broken or because the objects are +no longer isolated due to finalizer resurrection (see +:c:member:`~PyTypeObject.tp_finalize`). + +0. **Reachable** (not yet a cyclic isolate): All objects are in their normal, + reachable state. A reference cycle could exist, but an external reference + means the objects are not yet isolated. +#. **Unreachable but consistent:** The final reference from outside the cyclic + group of objects has been removed, causing the objects to become isolated + (thus a cyclic isolate is born). None of the group's objects have been + finalized or cleared yet. The cyclic isolate remains at this stage until + some future run of the garbage collector (not necessarily the next run + because the next run might not scan every object). +#. **Mix of finalized and not finalized:** Objects in a cyclic isolate are + finalized one at a time, which means that there is a period of time when the + cyclic isolate is composed of a mix of finalized and non-finalized objects. + Finalization order is unspecified, so it can appear random. A finalized + object must behave in a sane manner when non-finalized objects interact with + it, and a non-finalized object must be able to tolerate the finalization of + an arbitrary subset of its referents. +#. **All finalized:** All objects in a cyclic isolate are finalized before any + of them are cleared. +#. **Mix of finalized and cleared:** The objects can be cleared serially or + concurrently (but with the :term:`GIL` held); either way, some will finish + before others. A finalized object must be able to tolerate the clearing of + a subset of its referents. :pep:`442` calls this stage "cyclic trash". +#. **Leaked:** If a cyclic isolate still exists after all objects in the group + have been finalized and cleared, then the objects remain indefinitely + uncollectable (see :data:`gc.garbage`). It is a bug if a cyclic isolate + reaches this stage---it means the :c:member:`~PyTypeObject.tp_clear` methods + of the participating objects have failed to break the reference cycle as + required. + +If :c:member:`~PyTypeObject.tp_clear` did not exist, then Python would have no +way to safely break a reference cycle. Simply destroying an object in a cyclic +isolate would result in a dangling pointer, triggering undefined behavior when +an object referencing the destroyed object is itself destroyed. The clearing +step makes object destruction a two-phase process: first +:c:member:`~PyTypeObject.tp_clear` is called to partially destroy the objects +enough to detangle them from each other, then +:c:member:`~PyTypeObject.tp_dealloc` is called to complete the destruction. + +Unlike clearing, finalization is not a phase of destruction. A finalized +object must still behave properly by continuing to fulfill its design +contracts. An object's finalizer is allowed to execute arbitrary Python code, +and is even allowed to prevent the impending destruction by adding a reference. +The finalizer is only related to destruction by call order---if it runs, it runs +before destruction, which starts with :c:member:`~PyTypeObject.tp_clear` (if +called) and concludes with :c:member:`~PyTypeObject.tp_dealloc`. + +The finalization step is not necessary to safely reclaim the objects in a +cyclic isolate, but its existence makes it easier to design types that behave +in a sane manner when objects are cleared. Clearing an object might +necessarily leave it in a broken, partially destroyed state---it might be +unsafe to call any of the cleared object's methods or access any of its +attributes. With finalization, only finalized objects can possibly interact +with cleared objects; non-finalized objects are guaranteed to interact with +only non-cleared (but potentially finalized) objects. + +To summarize the possible interactions: + +* A non-finalized object might have references to or from non-finalized and + finalized objects, but not to or from cleared objects. +* A finalized object might have references to or from non-finalized, finalized, + and cleared objects. +* A cleared object might have references to or from finalized and cleared + objects, but not to or from non-finalized objects. + +Without any reference cycles, an object can be simply destroyed once its last +reference is deleted; the finalization and clearing steps are not necessary to +safely reclaim unused objects. However, it can be useful to automatically call +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` +before destruction anyway because type design is simplified when all objects +always experience the same series of events regardless of whether they +participated in a cyclic isolate. Python currently only calls +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` as +needed to destroy a cyclic isolate; this may change in a future version. + + +Functions +--------- + +To allocate and free memory, see :ref:`allocating-objects`. + + +.. c:function:: void PyObject_CallFinalizer(PyObject *op) + + Finalizes the object as described in :c:member:`~PyTypeObject.tp_finalize`. + Call this function (or :c:func:`PyObject_CallFinalizerFromDealloc`) instead + of calling :c:member:`~PyTypeObject.tp_finalize` directly because this + function may deduplicate multiple calls to :c:member:`!tp_finalize`. + Currently, calls are only deduplicated if the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may + change in the future. + + +.. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) + + Same as :c:func:`PyObject_CallFinalizer` but meant to be called at the + beginning of the object's destructor (:c:member:`~PyTypeObject.tp_dealloc`). + There must not be any references to the object. If the object's finalizer + resurrects the object, this function returns -1; no further destruction + should happen. Otherwise, this function returns 0 and destruction can + continue normally. + + .. seealso:: + + :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 64ae35daa703b8..61fa49f8681cce 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -376,6 +376,24 @@ The :ref:`default object allocator ` uses the If *p* is ``NULL``, no operation is performed. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`; use :c:func:`PyObject_GC_Del` instead. + + .. seealso:: + + * :c:func:`PyObject_GC_Del` is the equivalent of this function for memory + allocated by types that support garbage collection. + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:macro:`PyObject_New` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. _default-memory-allocators: diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 8bd8c107c98bdf..83de4248039949 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -12,6 +12,7 @@ object types. .. toctree:: allocation.rst + lifecycle.rst structures.rst typeobj.rst gcsupport.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index ec2867b0ce09ba..2176b8e492f306 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -151,14 +151,29 @@ Type Objects .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. + Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type + object. Uses Python's default memory allocation mechanism to allocate memory + for a new instance, zeros the memory, then initializes the memory as if by + calling :c:func:`PyObject_Init` or :c:func:`PyObject_InitVar`. + + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + For types that support garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set), this function behaves like + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar` (except the + memory is guaranteed to be zeroed before initialization), and should be + paired with :c:func:`PyObject_GC_Del` in :c:member:`~PyTypeObject.tp_free`. + Otherwise, it behaves like :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar` (except the memory is guaranteed to be zeroed + before initialization) and should be paired with :c:func:`PyObject_Free` in + :c:member:`~PyTypeObject.tp_free`. .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type + object. Creates a new instance using the type's + :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. .. c:function:: int PyType_Ready(PyTypeObject *type) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 3b9f07778d5ace..64353a8daca5f6 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -676,77 +676,122 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Free` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` + The destructor function should remove all references which the instance owns + (e.g., call :c:func:`Py_CLEAR`), free all memory buffers owned by the + instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to + free the object itself. + + No guarantees are made about when an object is destroyed, except: + + * Python will destroy an object immediately or some time after the final + reference to the object is deleted, unless its finalizer + (:c:member:`~PyTypeObject.tp_finalize`) subsequently resurrects the + object. + * An object will not be destroyed while it is being automatically finalized + (:c:member:`~PyTypeObject.tp_finalize`) or automatically cleared + (:c:member:`~PyTypeObject.tp_clear`). + + CPython currently destroys an object immediately from :c:func:`Py_DECREF` + when the new reference count is zero, but this may change in a future + version. + + It is recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is always + finalized before destruction. + + If the type supports garbage collection (the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set), the destructor should call :c:func:`PyObject_GC_UnTrack` before clearing any member fields. - .. code-block:: c + It is permissible to call :c:member:`~PyTypeObject.tp_clear` from + :c:member:`!tp_dealloc` to reduce code duplication and to guarantee that the + object is always cleared before destruction. Beware that + :c:member:`!tp_clear` might have already been called. - static void - foo_dealloc(PyObject *op) - { + If the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object (via + :c:func:`Py_DECREF`) after calling the type deallocator. See the example + code below.:: + + static void + foo_dealloc(PyObject *op) + { foo_object *self = (foo_object *) op; PyObject_GC_UnTrack(self); Py_CLEAR(self->ref); Py_TYPE(self)->tp_free(self); - } + } + + :c:member:`!tp_dealloc` must leave the exception status unchanged. If it + needs to call something that might raise an exception, the exception state + must be backed up first and restored later (after logging any exceptions + with :c:func:`PyErr_WriteUnraisable`). + + Example:: - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: + static void + foo_dealloc(PyObject *self) + { + PyObject *exc = PyErr_GetRaisedException(); - .. code-block:: c + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // self was resurrected. + goto done; + } - static void - foo_dealloc(PyObject *op) - { - PyTypeObject *tp = Py_TYPE(op); - // free references and buffers here - tp->tp_free(op); - Py_DECREF(tp); - } + PyTypeObject *tp = Py_TYPE(self); - .. warning:: + if (tp->tp_flags & Py_TPFLAGS_HAVE_GC) { + PyObject_GC_UnTrack(self); + } - In a garbage collected Python, :c:member:`!tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the - object becomes part of a refcount cycle, that cycle might be collected by - a garbage collection on any thread). This is not a problem for Python - API calls, since the thread on which :c:member:`!tp_dealloc` is called - with an :term:`attached thread state`. However, if the object being - destroyed in turn destroys objects from some other C or C++ library, care - should be taken to ensure that destroying those objects on the thread - which called :c:member:`!tp_dealloc` will not violate any assumptions of - the library. + // Optional, but convenient to avoid code duplication. + if (tp->tp_clear && tp->tp_clear(self) < 0) { + PyErr_WriteUnraisable(self); + } + + // Any additional destruction goes here. + + tp->tp_free(self); + self = NULL; // In case PyErr_WriteUnraisable() is called below. + + if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_CLEAR(tp); + } + + done: + // Optional, if something was called that might have raised an + // exception. + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + PyErr_SetRaisedException(exc); + } + + :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + with an :term:`attached thread state`. However, if the object being + destroyed in turn destroys objects from some other C library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. **Inheritance:** This field is inherited by subtypes. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset @@ -1137,11 +1182,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. + is set, memory for new instances (see :c:member:`~PyTypeObject.tp_alloc`) + must be allocated using :c:macro:`PyObject_GC_New` or + :c:func:`PyType_GenericAlloc` and deallocated (see + :c:member:`~PyTypeObject.tp_free`) using :c:func:`PyObject_GC_Del`. More + information in section :ref:`supporting-cycle-detection`. **Inheritance:** @@ -1478,6 +1523,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) heap-allocated superclass). If they do not, the type object may not be garbage-collected. + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + .. versionchanged:: 3.9 Heap-allocated types are expected to visit ``Py_TYPE(self)`` in @@ -1497,20 +1547,101 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + The purpose of this function is to break reference cycles that are causing a + :term:`cyclic isolate` so that the objects can be safely destroyed. A + cleared object is a partially destroyed object; the object is not obligated + to satisfy design invariants held during normal use. + + :c:member:`!tp_clear` does not need to delete references to objects that + can't participate in reference cycles, such as Python strings or Python + integers. However, it may be convenient to clear all references, and write + the type's :c:member:`~PyTypeObject.tp_dealloc` function to invoke + :c:member:`!tp_clear` to avoid code duplication. (Beware that + :c:member:`!tp_clear` might have already been called. Prefer calling + idempotent functions like :c:func:`Py_CLEAR`.) + + Any non-trivial cleanup should be performed in + :c:member:`~PyTypeObject.tp_finalize` instead of :c:member:`!tp_clear`. + + .. note:: + + If :c:member:`!tp_clear` fails to break a reference cycle then the + objects in the :term:`cyclic isolate` may remain indefinitely + uncollectable ("leak"). See :data:`gc.garbage`. + + .. note:: + + Referents (direct and indirect) might have already been cleared; they are + not guaranteed to be in a consistent state. + + .. note:: + + The :c:member:`~PyTypeObject.tp_clear` function can be called from any + thread. + + .. note:: + + An object is not guaranteed to be automatically cleared before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. + + This function differs from the destructor + (:c:member:`~PyTypeObject.tp_dealloc`) in the following ways: + + * The purpose of clearing an object is to remove references to other objects + that might participate in a reference cycle. The purpose of the + destructor, on the other hand, is a superset: it must release *all* + resources it owns, including references to objects that cannot participate + in a reference cycle (e.g., integers) as well as the object's own memory + (by calling :c:member:`~PyTypeObject.tp_free`). + * When :c:member:`!tp_clear` is called, other objects might still hold + references to the object being cleared. Because of this, + :c:member:`!tp_clear` must not deallocate the object's own memory + (:c:member:`~PyTypeObject.tp_free`). The destructor, on the other hand, + is only called when no (strong) references exist, and as such, must + safely destroy the object itself by deallocating it. + * :c:member:`!tp_clear` might never be automatically called. An object's + destructor, on the other hand, will be automatically called some time + after the object becomes unreachable (i.e., either there are no references + to the object or the object is a member of a :term:`cyclic isolate`). + + No guarantees are made about when, if, or how often Python automatically + clears an object, except: + + * Python will not automatically clear an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically clear an object if it has not been + automatically finalized (see :c:member:`~PyTypeObject.tp_finalize`). (If + the finalizer resurrected the object, the object may or may not be + automatically finalized again before it is cleared.) + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically clear it if any member of the cyclic isolate has not yet + been automatically finalized (:c:member:`~PyTypeObject.tp_finalize`). + * Python will not destroy an object until after any automatic calls to its + :c:member:`!tp_clear` function have returned. This ensures that the act + of breaking a reference cycle does not invalidate the ``self`` pointer + while :c:member:`!tp_clear` is still executing. + * Python will not automatically call :c:member:`!tp_clear` multiple times + concurrently. + + CPython currently only automatically clears objects as needed to break + reference cycles in a :term:`cyclic isolate`, but future versions might + clear objects regularly before their destruction. + + Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the + system must combine to break all reference cycles. This is subtle, and if + in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For + example, the tuple type does not implement a + :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove + that no reference cycle can be composed entirely of tuples. Therefore the + :c:member:`~PyTypeObject.tp_clear` functions of other types are responsible + for breaking any cycle containing a tuple. This isn't immediately obvious, + and there's rarely a good reason to avoid implementing + :c:member:`~PyTypeObject.tp_clear`. Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those @@ -1545,18 +1676,6 @@ and :c:data:`PyType_Type` effectively act as defaults.) PyObject_ClearManagedDict((PyObject*)self); - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. @@ -1569,6 +1688,10 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: richcmpfunc PyTypeObject.tp_richcompare @@ -1945,18 +2068,17 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). + Static subtypes inherit this slot, which will be + :c:func:`PyType_GenericAlloc` if inherited from :class:`object`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. + For heap subtypes, this field is always set to + :c:func:`PyType_GenericAlloc`. - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. + For static subtypes, this slot is inherited (see above). .. c:member:: newfunc PyTypeObject.tp_new @@ -2004,20 +2126,27 @@ and :c:data:`PyType_Type` effectively act as defaults.) void tp_free(void *self); - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. + This function must free the memory allocated by + :c:member:`~PyTypeObject.tp_alloc`. **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) + Static subtypes inherit this slot, which will be :c:func:`PyObject_Free` if + inherited from :class:`object`. Exception: If the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and it would inherit + :c:func:`PyObject_Free`, then this slot is not inherited but instead defaults + to :c:func:`PyObject_GC_Del`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. + For :ref:`heap subtypes `, this slot defaults to a deallocator suitable to match + :c:func:`PyType_GenericAlloc` and the value of the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. + For static subtypes, this slot is inherited (see above). .. c:member:: inquiry PyTypeObject.tp_is_gc @@ -2144,29 +2273,138 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize - An optional pointer to an instance finalization function. Its signature is:: + An optional pointer to an instance finalization function. This is the C + implementation of the :meth:`~object.__del__` special method. Its signature + is:: void tp_finalize(PyObject *self); - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. + The primary purpose of finalization is to perform any non-trivial cleanup + that must be performed before the object is destroyed, while the object and + any other objects it directly or indirectly references are still in a + consistent state. The finalizer is allowed to execute + arbitrary Python code. + + Before Python automatically finalizes an object, some of the object's direct + or indirect referents might have themselves been automatically finalized. + However, none of the referents will have been automatically cleared + (:c:member:`~PyTypeObject.tp_clear`) yet. + + Other non-finalized objects might still be using a finalized object, so the + finalizer must leave the object in a sane state (e.g., invariants are still + met). + + .. note:: + + After Python automatically finalizes an object, Python might start + automatically clearing (:c:member:`~PyTypeObject.tp_clear`) the object + and its referents (direct and indirect). Cleared objects are not + guaranteed to be in a consistent state; a finalized object must be able + to tolerate cleared referents. + + .. note:: + + An object is not guaranteed to be automatically finalized before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. It is + recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is + always finalized before destruction. + + .. note:: + + The :c:member:`~PyTypeObject.tp_finalize` function can be called from any + thread, although the :term:`GIL` will be held. - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: + .. note:: + + The :c:member:`!tp_finalize` function can be called during shutdown, + after some global variables have been deleted. See the documentation of + the :meth:`~object.__del__` method for details. + + When Python finalizes an object, it behaves like the following algorithm: + + #. Python might mark the object as *finalized*. Currently, Python always + marks objects whose type supports garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and never marks other types of + objects; this might change in a future version. + #. If the object is not marked as *finalized* and its + :c:member:`!tp_finalize` finalizer function is non-``NULL``, the + finalizer function is called. + #. If the finalizer function was called and the finalizer made the object + reachable (i.e., there is a reference to the object and it is not a + member of a :term:`cyclic isolate`), then the finalizer is said to have + *resurrected* the object. It is unspecified whether the finalizer can + also resurrect the object by adding a new reference to the object that + does not make it reachable, i.e., the object is (still) a member of a + cyclic isolate. + #. If the finalizer resurrected the object, the object's pending destruction + is canceled and the object's *finalized* mark might be removed if + present. Currently, Python never removes the *finalized* mark; this + might change in a future version. + + *Automatic finalization* refers to any finalization performed by Python + except via calls to :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. No guarantees are made about + when, if, or how often an object is automatically finalized, except: + + * Python will not automatically finalize an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically finalize an object if finalizing it would + not mark the object as *finalized*. Currently, this applies to objects + whose type does not support garbage collection, i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is not set. Such objects can still be + manually finalized by calling :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. + * Python will not automatically finalize any two members of a :term:`cyclic + isolate` concurrently. + * Python will not automatically finalize an object after it has + automatically cleared (:c:member:`~PyTypeObject.tp_clear`) the object. + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically finalize it after automatically clearing (see + :c:member:`~PyTypeObject.tp_clear`) any other member. + * Python will automatically finalize every member of a :term:`cyclic + isolate` before it automatically clears (see + :c:member:`~PyTypeObject.tp_clear`) any of them. + * If Python is going to automatically clear an object + (:c:member:`~PyTypeObject.tp_clear`), it will automatically finalize the + object first. + + Python currently only automatically finalizes objects that are members of a + :term:`cyclic isolate`, but future versions might finalize objects regularly + before their destruction. + + To manually finalize an object, do not call this function directly; call + :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc` instead. + + :c:member:`~PyTypeObject.tp_finalize` should leave the current exception + status unchanged. The recommended way to write a non-trivial finalizer is + to back up the exception at the beginning by calling + :c:func:`PyErr_GetRaisedException` and restore the exception at the end by + calling :c:func:`PyErr_SetRaisedException`. If an exception is encountered + in the middle of the finalizer, log and clear it with + :c:func:`PyErr_WriteUnraisable` or :c:func:`PyErr_FormatUnraisable`. For + example:: static void - local_finalize(PyObject *self) + foo_finalize(PyObject *self) { - /* Save the current exception, if any. */ + // Save the current exception, if any. PyObject *exc = PyErr_GetRaisedException(); - /* ... */ + // ... - /* Restore the saved exception. */ + if (do_something_that_might_raise() != success_indicator) { + PyErr_WriteUnraisable(self); + goto done; + } + + done: + // Restore the saved exception. This silently discards any exception + // raised above, so be sure to call PyErr_WriteUnraisable first if + // necessary. PyErr_SetRaisedException(exc); } @@ -2182,7 +2420,13 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. - .. seealso:: "Safe object finalization" (:pep:`442`) + .. seealso:: + + * :pep:`442`: "Safe object finalization" + * :ref:`life-cycle` for details about how this slot relates to other + slots. + * :c:func:`PyObject_CallFinalizer` + * :c:func:`PyObject_CallFinalizerFromDealloc` .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0b26e18efd7f1b..c5c7994f1262a9 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -355,6 +355,12 @@ Glossary tasks (see :mod:`asyncio`) associate each task with a context which becomes the current context whenever the task starts or resumes execution. + cyclic isolate + A subgroup of one or more objects that reference each other in a reference + cycle, but are not referenced by objects outside the group. The goal of + the :term:`cyclic garbage collector ` is to identify these groups and break the reference + cycles so that the memory can be reclaimed. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for From 6ce2045139e461d45e721745383245f484a7f167 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 19:22:24 +0200 Subject: [PATCH 159/713] [3.14] GH-133779: Fix finding pyconfig.h on Windows JIT builds (GH-134359) (cherry picked from commit 7ad90463df16e1938a3b6725b0f02af34cea372e) --- PCbuild/regen.targets | 3 +-- Tools/jit/_targets.py | 16 ++++++++-------- Tools/jit/build.py | 21 +++++++++++++++++---- configure | 2 +- configure.ac | 2 +- 5 files changed, 28 insertions(+), 16 deletions(-) diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 21de614e71ddce..742597f5cb5ebd 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -125,8 +125,7 @@ x86_64-pc-windows-msvc $(JITArgs) --debug - + diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 6ceb4404e74ce7..d0a1c081ffecc2 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -47,6 +47,7 @@ class _Target(typing.Generic[_S, _R]): debug: bool = False verbose: bool = False known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) + pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): @@ -57,13 +58,13 @@ def _get_nop(self) -> bytes: raise ValueError(f"NOP not defined for {self.triple}") return nop - def _compute_digest(self, out: pathlib.Path) -> str: + def _compute_digest(self) -> str: hasher = hashlib.sha256() hasher.update(self.triple.encode()) hasher.update(self.debug.to_bytes()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) - hasher.update((out / "pyconfig.h").read_bytes()) + hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes()) for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): for filename in filenames: hasher.update(pathlib.Path(dirpath, filename).read_bytes()) @@ -125,7 +126,7 @@ async def _compile( f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", "-D_Py_JIT", - "-I.", + f"-I{self.pyconfig_dir}", f"-I{CPYTHON / 'Include'}", f"-I{CPYTHON / 'Include' / 'internal'}", f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", @@ -193,20 +194,19 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: def build( self, - out: pathlib.Path, *, comment: str = "", force: bool = False, - stencils_h: str = "jit_stencils.h", + jit_stencils: pathlib.Path, ) -> None: """Build jit_stencils.h in the given directory.""" + jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" request = "Please report any issues you encounter.".center(len(warning)) outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) - digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / stencils_h + digest = f"// {self._compute_digest()}\n" if ( not force and jit_stencils.exists() @@ -214,7 +214,7 @@ def build( ): return stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) - jit_stencils_new = out / "jit_stencils.h.new" + jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: file.write(digest) diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 49b08f477dbed7..1afd0c76bad1d2 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,7 +8,6 @@ import _targets if __name__ == "__main__": - out = pathlib.Path.cwd().resolve() comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -23,6 +22,20 @@ parser.add_argument( "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" ) + parser.add_argument( + "-o", + "--output-dir", + help="where to output generated files", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) + parser.add_argument( + "-p", + "--pyconfig-dir", + help="where to find pyconfig.h", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) parser.add_argument( "-v", "--verbose", action="store_true", help="echo commands as they are run" ) @@ -31,13 +44,13 @@ target.debug = args.debug target.force = args.force target.verbose = args.verbose + target.pyconfig_dir = args.pyconfig_dir target.build( - out, comment=comment, - stencils_h=f"jit_stencils-{target.triple}.h", force=args.force, + jit_stencils=args.output_dir / f"jit_stencils-{target.triple}.h", ) - jit_stencils_h = out / "jit_stencils.h" + jit_stencils_h = args.output_dir / "jit_stencils.h" lines = [f"// {comment}\n"] guard = "#if" for target in args.target: diff --git a/configure b/configure index a43f1f84d046b8..5d21d06014a238 100755 --- a/configure +++ b/configure @@ -10863,7 +10863,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir ." JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : diff --git a/configure.ac b/configure.ac index e3f39f19fc23e1..8613aa58257f5b 100644 --- a/configure.ac +++ b/configure.ac @@ -2776,7 +2776,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir ."]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) AS_VAR_IF([Py_DEBUG], [true], From efdaae5b356a158cf6028844d08b0efd1c5313cb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 19:43:09 +0200 Subject: [PATCH 160/713] [3.14] gh-86802: Fix asyncio memory leak; shielded task exceptions log once through the exception handler (gh-134331) (gh-134343) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit f695eca60cfc53cf3322323082652037d6d0cfef) Co-authored-by: Christian Harries <68507104+ChristianHrs@users.noreply.github.com> Co-authored-by: Łukasz Langa --- Lib/asyncio/tasks.py | 36 +++++++++++++---- Lib/test/test_asyncio/test_tasks.py | 40 +++++++++++++++++++ ...5-05-20-15-13-43.gh-issue-86802.trF7TM.rst | 3 ++ 3 files changed, 72 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-20-15-13-43.gh-issue-86802.trF7TM.rst diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 888615f8e5e1b3..fbd5c39a7c56ac 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -908,6 +908,25 @@ def _done_callback(fut, cur_task=cur_task): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -953,14 +972,11 @@ def shield(arg): else: cur_task = None - def _inner_done_callback(inner, cur_task=cur_task): - if cur_task is not None: - futures.future_discard_from_awaited_by(inner, cur_task) + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -972,10 +988,16 @@ def _inner_done_callback(inner, cur_task=cur_task): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 44498ef790e450..f6f976f213ac02 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2116,6 +2116,46 @@ def test_shield_cancel_outer(self): self.assertTrue(outer.cancelled()) self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + def test_shield_shortcut(self): fut = self.new_future(self.loop) fut.set_result(42) diff --git a/Misc/NEWS.d/next/Library/2025-05-20-15-13-43.gh-issue-86802.trF7TM.rst b/Misc/NEWS.d/next/Library/2025-05-20-15-13-43.gh-issue-86802.trF7TM.rst new file mode 100644 index 00000000000000..d3117b16f04436 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-20-15-13-43.gh-issue-86802.trF7TM.rst @@ -0,0 +1,3 @@ +Fixed asyncio memory leak in cancelled shield tasks. For shielded tasks +where the shield was cancelled, log potential exceptions through the +exception handler. Contributed by Christian Harries. From 2d90cc8b35d195f5330120ee3d606c5961228cdf Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 20:49:29 +0200 Subject: [PATCH 161/713] [3.14] gh-106213: Shorten Emscripten wasm-gc trampoline by a little (GH-133984) (#134376) Using the if instruction results in slightly shorter trampoline code. (cherry picked from commit 3b7888bf3d43b903f0a7ebd16f39d8bb61dfbb9e) Co-authored-by: Hood Chatham --- Python/emscripten_trampoline.c | 58 ++++++++++++---------------------- 1 file changed, 21 insertions(+), 37 deletions(-) diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index a7bb685bf3dc6d..cc5047d6bda224 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -35,7 +35,7 @@ EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { // (type $type1 (func (param i32) (result i32))) // (type $type2 (func (param i32 i32) (result i32))) // (type $type3 (func (param i32 i32 i32) (result i32))) -// (type $blocktype (func (param i32) (result))) +// (type $blocktype (func (param) (result))) // (table $funcs (import "e" "t") 0 funcref) // (export "f" (func $f)) // (func $f (param $fptr i32) (result i32) @@ -44,36 +44,28 @@ EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { // table.get $funcs // local.tee $fref // ref.test $type3 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b +// if $blocktype // i32.const 3 // return -// ) +// end // local.get $fref // ref.test $type2 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b +// if $blocktype // i32.const 2 // return -// ) +// end // local.get $fref // ref.test $type1 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b +// if $blocktype // i32.const 1 // return -// ) +// end // local.get $fref // ref.test $type0 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b +// if $blocktype // i32.const 0 // return -// ) +// end // i32.const -1 // ) // ) @@ -88,13 +80,13 @@ function getPyEMCountArgsPtr() { const code = new Uint8Array([ 0x00, 0x61, 0x73, 0x6d, // \0asm magic number 0x01, 0x00, 0x00, 0x00, // version 1 - 0x01, 0x1b, // Type section, body is 0x1b bytes + 0x01, 0x1a, // Type section, body is 0x1a bytes 0x05, // 6 entries - 0x60, 0x00, 0x01, 0x7f, // (type $type0 (func (param) (result i32))) - 0x60, 0x01, 0x7f, 0x01, 0x7f, // (type $type1 (func (param i32) (result i32))) - 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, // (type $type2 (func (param i32 i32) (result i32))) - 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, // (type $type3 (func (param i32 i32 i32) (result i32))) - 0x60, 0x01, 0x7f, 0x00, // (type $blocktype (func (param i32) (result))) + 0x60, 0x00, 0x01, 0x7f, // (type $type0 (func (param) (result i32))) + 0x60, 0x01, 0x7f, 0x01, 0x7f, // (type $type1 (func (param i32) (result i32))) + 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, // (type $type2 (func (param i32 i32) (result i32))) + 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, // (type $type3 (func (param i32 i32 i32) (result i32))) + 0x60, 0x00, 0x00, // (type $blocktype (func (param) (result))) 0x02, 0x09, // Import section, 0x9 byte body 0x01, // 1 import (table $funcs (import "e" "t") 0 funcref) 0x01, 0x65, // "e" @@ -110,44 +102,36 @@ function getPyEMCountArgsPtr() { 0x00, // a function 0x00, // at index 0 - 0x0a, 0x44, // Code section, - 0x01, 0x42, // one entry of length 50 + 0x0a, 56, // Code section, + 0x01, 54, // one entry of length 54 0x01, 0x01, 0x70, // one local of type funcref // Body of the function 0x20, 0x00, // local.get $fptr 0x25, 0x00, // table.get $funcs 0x22, 0x01, // local.tee $fref 0xfb, 0x14, 0x03, // ref.test $type3 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b + 0x04, 0x04, // if (type $blocktype) 0x41, 0x03, // i32.const 3 0x0f, // return 0x0b, // end block 0x20, 0x01, // local.get $fref 0xfb, 0x14, 0x02, // ref.test $type2 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b + 0x04, 0x04, // if (type $blocktype) 0x41, 0x02, // i32.const 2 0x0f, // return 0x0b, // end block 0x20, 0x01, // local.get $fref 0xfb, 0x14, 0x01, // ref.test $type1 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b + 0x04, 0x04, // if (type $blocktype) 0x41, 0x01, // i32.const 1 0x0f, // return 0x0b, // end block 0x20, 0x01, // local.get $fref 0xfb, 0x14, 0x00, // ref.test $type0 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b + 0x04, 0x04, // if (type $blocktype) 0x41, 0x00, // i32.const 0 0x0f, // return 0x0b, // end block From a2950d50ccd1e12f4e26f888ef66392dad886606 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 20:59:00 +0200 Subject: [PATCH 162/713] [3.14] gh-131357: Add tests for zero-sized bytes objects in test_bytes.py (GH-134234) (#134378) gh-131357: Add tests for zero-sized bytes objects in test_bytes.py (GH-134234) (cherry picked from commit 306f9e04e50c80f7be2a38fc3b64accf59f5164c) Co-authored-by: abstractedfox Co-authored-by: Victor Stinner --- Lib/test/test_capi/test_bytes.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 5b61c73381542d..86d5692893fc98 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -22,6 +22,7 @@ def test_check(self): # Test PyBytes_Check() check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertTrue(check(BytesSubclass(b'abc'))) @@ -36,6 +37,7 @@ def test_checkexact(self): # Test PyBytes_CheckExact() check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertFalse(check(BytesSubclass(b'abc'))) @@ -108,6 +110,7 @@ def test_asstring(self): self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') + self.assertEqual(asstring(b'', 1), b'\0') self.assertRaises(TypeError, asstring, 'abc', 0) self.assertRaises(TypeError, asstring, object(), 0) @@ -120,6 +123,7 @@ def test_asstringandsize(self): self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(asstringandsize(b'', 1), (b'\0', 0)) self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) self.assertRaises(TypeError, asstringandsize, 'abc', 0) @@ -163,6 +167,7 @@ def test_concat(self, concat=None): self.assertEqual(concat(b'', bytearray(b'def')), b'def') self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') + self.assertEqual(concat(b'', b''), b'') self.assertEqual(concat(b'abc', b'def', True), b'abcdef') self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') From 6f1f8714839aaeadfbe646373e9f1bbf15875155 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 21:07:08 +0200 Subject: [PATCH 163/713] [3.14] gh-127146: xfail more Emscripten stack overflows (GH-134358) (#134382) Adds some additional test xfails for Emscripten stack overflows. Also corrects a test skip for test_io. (cherry picked from commit 91e6a58e2d6fd23e886135457e28dfa980ec49ed) Co-authored-by: Hood Chatham --- Lib/test/test_builtin.py | 1 + Lib/test/test_capi/test_misc.py | 2 ++ Lib/test/test_descr.py | 1 + Lib/test/test_exceptions.py | 1 + Lib/test/test_io.py | 2 +- 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 31597a320d418a..f1f877c47a1e7a 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -1120,6 +1120,7 @@ def test_filter_pickle(self): self.check_iter_pickle(f1, list(f2), proto) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_filter_dealloc(self): # Tests recursive deallocation of nested filter objects using the diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a597f23a992e7b..f74694a7a745a4 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -412,10 +412,12 @@ def test_trashcan_subclass(self): L = MyList((L,)) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() def test_trashcan_python_class1(self): self.do_test_trashcan_python_class(list) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() def test_trashcan_python_class2(self): from _testcapi import MyList self.do_test_trashcan_python_class(MyList) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 76937432a43037..14026531e22333 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -4523,6 +4523,7 @@ class Oops(object): del o @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_wrapper_segfault(self): # SF 927248: deeply nested wrappers could cause stack overflow diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index d177e3dc0f5007..cfd4a3e87695f5 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -1429,6 +1429,7 @@ def g(): self.assertIn("maximum recursion depth exceeded", str(exc)) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @cpython_only @support.requires_resource('cpu') def test_trashcan_recursion(self): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5a8f1949baaa98..90680c6d47ab41 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -572,7 +572,7 @@ def do_test(test, obj, abilities): for [test, abilities] in tests: with self.subTest(test): if test == pipe_writer and not threading_helper.can_start_thread: - skipTest() + self.skipTest("Need threads") with test() as obj: do_test(test, obj, abilities) From 899ce7d390d5e37088a841cdcefd142013355e4c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 21:28:03 +0200 Subject: [PATCH 164/713] [3.14] gh-85045: clarified that the underlying buffer of a TextIOBase can be a RawIOBase (GH-134372) (#134374) gh-85045: clarified that the underlying buffer of a TextIOBase can be a RawIOBase (GH-134372) Added a clarification that the underlying binary buffer of a TextIOBase can be a BufferedIOBase OR a RawIOBase (cherry picked from commit 36eb711d2f26849214774a017fe8c8a5be3eec30) Co-authored-by: Alex Kautz --- Doc/library/io.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 3aa2f35f05eba0..08c76da3d8c00a 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -894,9 +894,10 @@ Text I/O .. attribute:: buffer - The underlying binary buffer (a :class:`BufferedIOBase` instance) that - :class:`TextIOBase` deals with. This is not part of the - :class:`TextIOBase` API and may not exist in some implementations. + The underlying binary buffer (a :class:`BufferedIOBase` + or :class:`RawIOBase` instance) that :class:`TextIOBase` deals with. + This is not part of the :class:`TextIOBase` API and may not exist + in some implementations. .. method:: detach() From 379805d03c7e2b3a19484c8247fad8da86b5a157 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 22:15:23 +0200 Subject: [PATCH 165/713] [3.14] gh-128066: Properly handle history file writes for RO fs on PyREPL (gh-134380) (gh-134385) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit c91ad5da9d92eac4718e4da8d53689c3cc24535e) Co-authored-by: Chris Patti Co-authored-by: Łukasz Langa --- Lib/_pyrepl/simple_interact.py | 2 ++ Lib/site.py | 8 +++++++- .../2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst | 3 +++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index c23894a34b8b53..965b853c34b392 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -31,6 +31,7 @@ import sys import code import warnings +import errno from .readline import _get_reader, multiline_input, append_history_file @@ -153,6 +154,7 @@ def maybe_run_command(statement: str) -> bool: append_history_file() except (FileNotFoundError, PermissionError, OSError) as e: warnings.warn(f"failed to open the history file for writing: {e}") + input_n += 1 except KeyboardInterrupt: r = _get_reader() diff --git a/Lib/site.py b/Lib/site.py index 5c38b1b17d5abd..f93271971594d8 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import _io as io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -578,10 +579,15 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise atexit.register(write_history) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst new file mode 100644 index 00000000000000..f78190276851b4 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst @@ -0,0 +1,3 @@ +Fixes an edge case where PyREPL improperly threw an error when Python is +invoked on a read only filesystem while trying to write history file +entries. From 04829d4d87659da7f2239cbe15000824bfb9c2d3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 23:01:45 +0200 Subject: [PATCH 166/713] [3.14] gh-134209: use heap-allocated memory in `_curses.window.{instr,getstr}` (GH-134283) (#134391) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-134209: use heap-allocated memory in `_curses.window.{instr,getstr}` (GH-134283) * made curses buffer heap allocated instead of stack * change docs to explicitly mention the max buffer size * changing GetStr() function to behave similarly too * Update Doc/library/curses.rst * Update instr with proper return error handling * Update Modules/_cursesmodule.c * change to strlen and better memory safety * change from const int to Py_ssize_t * add mem allocation guard * update versionchanged to mention it was an increase. * explicitly use versionchanged 3.14 as that is its own branch now. TESTED: `python -m test -u curses test_curses` --------- (cherry picked from commit aadda87b3d3d99cb9e8c8791bb9715a3f0209195) Co-authored-by: tigerding <43339228+zydtiger@users.noreply.github.com> Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> Co-authored-by: Gregory P. Smith Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/curses.rst | 8 ++ ...-05-19-20-59-06.gh-issue-134209.anhTcF.rst | 3 + Modules/_cursesmodule.c | 102 ++++++++++++------ 3 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-19-20-59-06.gh-issue-134209.anhTcF.rst diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 137504c51b4358..5ec23b61396773 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -988,6 +988,10 @@ the following methods and attributes: window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.getyx() @@ -1079,6 +1083,10 @@ the following methods and attributes: current cursor position, or at *y*, *x* if specified. Attributes are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.is_linetouched(line) diff --git a/Misc/NEWS.d/next/Library/2025-05-19-20-59-06.gh-issue-134209.anhTcF.rst b/Misc/NEWS.d/next/Library/2025-05-19-20-59-06.gh-issue-134209.anhTcF.rst new file mode 100644 index 00000000000000..f985872f3c975e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-20-59-06.gh-issue-134209.anhTcF.rst @@ -0,0 +1,3 @@ +:mod:`curses`: The :meth:`curses.window.instr` and :meth:`curses.window.getstr` +methods now allocate their internal buffer on the heap instead of the stack; +in addition, the max buffer size is increased from 1023 to 2047. diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index cd185bc2b02ea5..f6e366ef7dec8b 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1687,7 +1687,7 @@ _curses.window.getstr x: int X-coordinate. ] - n: int = 1023 + n: int = 2047 Maximal number of characters. / @@ -1700,62 +1700,80 @@ PyCursesWindow_GetStr(PyObject *op, PyObject *args) PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; + int rtn; + + /* could make the buffer size larger/dynamic */ + Py_ssize_t max_buf_size = 2048; + PyObject *result = PyBytes_FromStringAndSize(NULL, max_buf_size); + if (result == NULL) + return NULL; + char *buf = PyBytes_AS_STRING(result); switch (PyTuple_Size(args)) { case 0: Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win,rtn, 1023); + rtn = wgetnstr(self->win, buf, max_buf_size - 1); Py_END_ALLOW_THREADS break; case 1: if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; + goto error; if (n < 0) { PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; + goto error; } Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win, rtn, Py_MIN(n, 1023)); + rtn = wgetnstr(self->win, buf, Py_MIN(n, max_buf_size - 1)); Py_END_ALLOW_THREADS break; case 2: if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; + goto error; Py_BEGIN_ALLOW_THREADS #ifdef STRICT_SYSV_CURSES - rtn2 = wmove(self->win,y,x)==ERR ? ERR : wgetnstr(self->win, rtn, 1023); + rtn = wmove(self->win,y,x)==ERR ? ERR : wgetnstr(self->win, rtn, max_buf_size - 1); #else - rtn2 = mvwgetnstr(self->win,y,x,rtn, 1023); + rtn = mvwgetnstr(self->win,y,x,buf, max_buf_size - 1); #endif Py_END_ALLOW_THREADS break; case 3: if (!PyArg_ParseTuple(args,"iii;y,x,n", &y, &x, &n)) - return NULL; + goto error; if (n < 0) { PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; + goto error; } #ifdef STRICT_SYSV_CURSES Py_BEGIN_ALLOW_THREADS - rtn2 = wmove(self->win,y,x)==ERR ? ERR : - wgetnstr(self->win, rtn, Py_MIN(n, 1023)); + rtn = wmove(self->win,y,x)==ERR ? ERR : + wgetnstr(self->win, rtn, Py_MIN(n, max_buf_size - 1)); Py_END_ALLOW_THREADS #else Py_BEGIN_ALLOW_THREADS - rtn2 = mvwgetnstr(self->win, y, x, rtn, Py_MIN(n, 1023)); + rtn = mvwgetnstr(self->win, y, x, buf, Py_MIN(n, max_buf_size - 1)); Py_END_ALLOW_THREADS #endif break; default: PyErr_SetString(PyExc_TypeError, "getstr requires 0 to 3 arguments"); + goto error; + } + + if (rtn == ERR) { + Py_DECREF(result); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + if (_PyBytes_Resize(&result, strlen(buf)) < 0) { return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + + return result; + +error: + Py_DECREF(result); + return NULL; } /*[clinic input] @@ -1889,7 +1907,7 @@ _curses.window.instr x: int X-coordinate. ] - n: int = 1023 + n: int = 2047 Maximal number of characters. / @@ -1906,43 +1924,61 @@ PyCursesWindow_InStr(PyObject *op, PyObject *args) PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; + int rtn; + + /* could make the buffer size larger/dynamic */ + Py_ssize_t max_buf_size = 2048; + PyObject *result = PyBytes_FromStringAndSize(NULL, max_buf_size); + if (result == NULL) + return NULL; + char *buf = PyBytes_AS_STRING(result); switch (PyTuple_Size(args)) { case 0: - rtn2 = winnstr(self->win,rtn, 1023); + rtn = winnstr(self->win, buf, max_buf_size - 1); break; case 1: if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; + goto error; if (n < 0) { PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; + goto error; } - rtn2 = winnstr(self->win, rtn, Py_MIN(n, 1023)); + rtn = winnstr(self->win, buf, Py_MIN(n, max_buf_size - 1)); break; case 2: if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; - rtn2 = mvwinnstr(self->win,y,x,rtn,1023); + goto error; + rtn = mvwinnstr(self->win, y, x, buf, max_buf_size - 1); break; case 3: if (!PyArg_ParseTuple(args, "iii;y,x,n", &y, &x, &n)) - return NULL; + goto error; if (n < 0) { PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; + goto error; } - rtn2 = mvwinnstr(self->win, y, x, rtn, Py_MIN(n,1023)); + rtn = mvwinnstr(self->win, y, x, buf, Py_MIN(n, max_buf_size - 1)); break; default: PyErr_SetString(PyExc_TypeError, "instr requires 0 or 3 arguments"); + goto error; + } + + if (rtn == ERR) { + Py_DECREF(result); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + + if (_PyBytes_Resize(&result, strlen(buf)) < 0) { return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + + return result; + +error: + Py_DECREF(result); + return NULL; } /*[clinic input] From 69710b708705f9b774a5955f0ff9ad6c1e74d32e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 23:12:02 +0200 Subject: [PATCH 167/713] [3.14] gh-134215: PyREPL: Do not show underscored modules by default during autocompletion (gh-134267) (gh-134388) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit a3a3cf6d157948ed64ae837d6310b933a39a2493) Co-authored-by: Kevin Hernández Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Tomas R. Co-authored-by: Łukasz Langa --- Lib/_pyrepl/_module_completer.py | 15 +++++-- Lib/test/test_pyrepl/test_pyrepl.py | 45 ++++++++++++++++++- Misc/ACKS | 1 + ...-05-19-14-57-46.gh-issue-134215.sbdDK6.rst | 1 + 4 files changed, 57 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-05-19-14-57-46.gh-issue-134215.sbdDK6.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 0606797226d1e0..9aafb55090e2ce 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -81,8 +81,10 @@ def find_modules(self, path: str, prefix: str) -> list[str]: def _find_modules(self, path: str, prefix: str) -> list[str]: if not path: # Top-level import (e.g. `import foo`` or `from foo`)` - builtin_modules = [name for name in sys.builtin_module_names if name.startswith(prefix)] - third_party_modules = [name for _, name, _ in self.global_cache if name.startswith(prefix)] + builtin_modules = [name for name in sys.builtin_module_names + if self.is_suggestion_match(name, prefix)] + third_party_modules = [module.name for module in self.global_cache + if self.is_suggestion_match(module.name, prefix)] return sorted(builtin_modules + third_party_modules) if path.startswith('.'): @@ -98,7 +100,14 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: if mod_info.ispkg and mod_info.name == segment] modules = self.iter_submodules(modules) return [module.name for module in modules - if module.name.startswith(prefix)] + if self.is_suggestion_match(module.name, prefix)] + + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: + if prefix: + return module_name.startswith(prefix) + # For consistency with attribute completion, which + # does not suggest private attributes unless requested. + return not module_name.startswith("_") def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: """Iterate over all submodules of the given parent modules.""" diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 59f5d1f893f9fd..29762232d43b89 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -8,6 +8,7 @@ import subprocess import sys import tempfile +from pkgutil import ModuleInfo from unittest import TestCase, skipUnless, skipIf from unittest.mock import patch from test.support import force_not_colorized, make_clean_env, Py_DEBUG @@ -959,6 +960,46 @@ def test_import_completions(self): output = reader.readline() self.assertEqual(output, expected) + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True)]) + @patch("sys.builtin_module_names", ()) + def test_private_completions(self): + cases = ( + # Return public methods by default + ("import \t\n", "import public"), + ("from \t\n", "from public"), + # Return private methods if explicitly specified + ("import _\t\n", "import _private"), + ("from _\t\n", "from _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch( + "_pyrepl._module_completer.ModuleCompleter.iter_submodules", + lambda *_: [ + ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True), + ], + ) + def test_sub_module_private_completions(self): + cases = ( + # Return public methods by default + ("from foo import \t\n", "from foo import public"), + # Return private methods if explicitly specified + ("from foo import _\t\n", "from foo import _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + def test_builtin_completion_top_level(self): import importlib # Make iter_modules() search only the standard library. @@ -991,8 +1032,8 @@ def test_relative_import_completions(self): output = reader.readline() self.assertEqual(output, expected) - @patch("pkgutil.iter_modules", lambda: [(None, 'valid_name', None), - (None, 'invalid-name', None)]) + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "valid_name", True), + ModuleInfo(None, "invalid-name", True)]) def test_invalid_identifiers(self): # Make sure modules which are not valid identifiers # are not suggested as those cannot be imported via 'import'. diff --git a/Misc/ACKS b/Misc/ACKS index 610dcf9f4238de..2b5748ceb6ad26 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -763,6 +763,7 @@ Chris Herborth Ivan Herman Jürgen Hermann Joshua Jay Herman +Kevin Hernandez Gary Herron Ernie Hershey Thomas Herve diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-05-19-14-57-46.gh-issue-134215.sbdDK6.rst b/Misc/NEWS.d/next/Tools-Demos/2025-05-19-14-57-46.gh-issue-134215.sbdDK6.rst new file mode 100644 index 00000000000000..546ed2a56b6695 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-05-19-14-57-46.gh-issue-134215.sbdDK6.rst @@ -0,0 +1 @@ +:term:`REPL` import autocomplete only suggests private modules when explicitly specified. From 81c348b6dadc6b4f035b80e508dd0ec583e54d2f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 20 May 2025 23:47:12 +0200 Subject: [PATCH 168/713] GH-130727: Avoid race condition in _wmimodule by copying shared data (GH-134313) (cherry picked from commit e4fbfb12889013fd52565cd2598a366754cb677b) Co-authored-by: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- ...-05-20-21-43-20.gh-issue-130727.-69t4D.rst | 2 ++ PC/_wmimodule.cpp | 22 +++++++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) create mode 100644 Misc/NEWS.d/next/Windows/2025-05-20-21-43-20.gh-issue-130727.-69t4D.rst diff --git a/Misc/NEWS.d/next/Windows/2025-05-20-21-43-20.gh-issue-130727.-69t4D.rst b/Misc/NEWS.d/next/Windows/2025-05-20-21-43-20.gh-issue-130727.-69t4D.rst new file mode 100644 index 00000000000000..dc10b3e62c8d4a --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2025-05-20-21-43-20.gh-issue-130727.-69t4D.rst @@ -0,0 +1,2 @@ +Fix a race in internal calls into WMI that can result in an "invalid handle" +exception under high load. Patch by Chris Eibl. diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index b6efb3e4a207b4..30d61c86587fbe 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -57,11 +57,11 @@ _query_thread(LPVOID param) IEnumWbemClassObject* enumerator = NULL; HRESULT hr = S_OK; BSTR bstrQuery = NULL; - struct _query_data *data = (struct _query_data*)param; + _query_data data = *(struct _query_data*)param; // gh-125315: Copy the query string first, so that if the main thread gives // up on waiting we aren't left with a dangling pointer (and a likely crash) - bstrQuery = SysAllocString(data->query); + bstrQuery = SysAllocString(data.query); if (!bstrQuery) { hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); } @@ -71,7 +71,7 @@ _query_thread(LPVOID param) } if (FAILED(hr)) { - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); if (bstrQuery) { SysFreeString(bstrQuery); } @@ -96,7 +96,7 @@ _query_thread(LPVOID param) IID_IWbemLocator, (LPVOID *)&locator ); } - if (SUCCEEDED(hr) && !SetEvent(data->initEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.initEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -105,7 +105,7 @@ _query_thread(LPVOID param) NULL, NULL, 0, NULL, 0, 0, &services ); } - if (SUCCEEDED(hr) && !SetEvent(data->connectEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.connectEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -143,7 +143,7 @@ _query_thread(LPVOID param) if (FAILED(hr) || got != 1 || !value) { continue; } - if (!startOfEnum && !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL)) { + if (!startOfEnum && !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL)) { hr = HRESULT_FROM_WIN32(GetLastError()); break; } @@ -171,10 +171,10 @@ _query_thread(LPVOID param) DWORD cbStr1, cbStr2; cbStr1 = (DWORD)(wcslen(propName) * sizeof(propName[0])); cbStr2 = (DWORD)(wcslen(propStr) * sizeof(propStr[0])); - if (!WriteFile(data->writePipe, propName, cbStr1, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"=", 2, &written, NULL) || - !WriteFile(data->writePipe, propStr, cbStr2, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL) + if (!WriteFile(data.writePipe, propName, cbStr1, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"=", 2, &written, NULL) || + !WriteFile(data.writePipe, propStr, cbStr2, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL) ) { hr = HRESULT_FROM_WIN32(GetLastError()); } @@ -200,7 +200,7 @@ _query_thread(LPVOID param) locator->Release(); } CoUninitialize(); - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); return (DWORD)hr; } From e379a71811d370731f4dc814e5628d14952713d5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 02:20:53 +0200 Subject: [PATCH 169/713] [3.14] gh-91048: Fix error path result in _remote_debugging_module (GH-134347) (#134399) --- Modules/_remote_debugging_module.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 9314ddd9bed5d7..f9f77a0814b4ce 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -1556,7 +1556,7 @@ get_stack_trace(PyObject* self, PyObject* args) &address_of_current_frame) < 0) { - Py_DECREF(result); + Py_CLEAR(result); goto result_err; } @@ -1565,7 +1565,7 @@ get_stack_trace(PyObject* self, PyObject* args) } if (PyList_Append(result, frame_info) == -1) { - Py_DECREF(result); + Py_CLEAR(result); goto result_err; } From af428a3894745652d48ad0d62c74ba8d2f5b451a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 03:58:48 +0200 Subject: [PATCH 170/713] [3.14] gh-72680: Fix false positives when using zipfile.is_zipfile() (GH-134250) (#134401) gh-72680: Fix false positives when using zipfile.is_zipfile() (GH-134250) bpo-28494: Improve zipfile.is_zipfile reliability The zipfile.is_zipfile function would only search for the EndOfZipfile section header. This failed to correctly identify non-zipfiles that contained this header. Now the zipfile.is_zipfile function verifies the first central directory entry. Changes: * Extended zipfile.is_zipfile to verify zipfile catalog * Added tests to validate failure of binary non-zipfiles * Reuse 'concat' handling for is_zipfile (cherry picked from commit 1298511b41ec0f9be925c12f3830e94fe8f7e7dc) Co-authored-by: Tim Hatch Co-authored-by: John Jolly --- Lib/test/test_zipfile/test_core.py | 19 ++++++++ Lib/zipfile/__init__.py | 48 +++++++++++++------ .../2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst | 1 + 3 files changed, 54 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index 43056978848c03..e93603998f979e 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1991,6 +1991,25 @@ def test_is_zip_erroneous_file(self): self.assertFalse(zipfile.is_zipfile(fp)) fp.seek(0, 0) self.assertFalse(zipfile.is_zipfile(fp)) + # - passing non-zipfile with ZIP header elements + # data created using pyPNG like so: + # d = [(ord('P'), ord('K'), 5, 6), (ord('P'), ord('K'), 6, 6)] + # w = png.Writer(1,2,alpha=True,compression=0) + # f = open('onepix.png', 'wb') + # w.write(f, d) + # w.close() + data = (b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" + b"\x00\x02\x08\x06\x00\x00\x00\x99\x81\xb6'\x00\x00\x00\x15I" + b"DATx\x01\x01\n\x00\xf5\xff\x00PK\x05\x06\x00PK\x06\x06\x07" + b"\xac\x01N\xc6|a\r\x00\x00\x00\x00IEND\xaeB`\x82") + # - passing a filename + with open(TESTFN, "wb") as fp: + fp.write(data) + self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a file-like object + fp = io.BytesIO() + fp.write(data) + self.assertFalse(zipfile.is_zipfile(fp)) def test_damaged_zipfile(self): """Check that zipfiles with missing bytes at the end raise BadZipFile.""" diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 894b4d37233923..18caeb3e04a2b5 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -234,8 +234,19 @@ def strip(cls, data, xids): def _check_zipfile(fp): try: - if _EndRecData(fp): - return True # file has correct magic number + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + # Central directory is on the same disk + fp.seek(sum(_handle_prepended_data(endrec))) + if endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number except OSError: pass return False @@ -258,6 +269,22 @@ def is_zipfile(filename): pass return result +def _handle_prepended_data(endrec, debug=0): + size_cd = endrec[_ECD_SIZE] # bytes in central directory + offset_cd = endrec[_ECD_OFFSET] # offset of central directory + + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd + if endrec[_ECD_SIGNATURE] == stringEndArchive64: + # If Zip64 extension structures are present, account for them + concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + + if debug > 2: + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) + + return offset_cd, concat + def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec @@ -1501,28 +1528,21 @@ def _RealGetContents(self): raise BadZipFile("File is not a zip file") if self.debug > 1: print(endrec) - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # "concat" is zero, unless zip was concatenated to another file - concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + offset_cd, concat = _handle_prepended_data(endrec, self.debug) + + # self.start_dir: Position of start of central directory + self.start_dir = offset_cd + concat # store the offset to the beginning of data for the # .data_offset property self._data_offset = concat - if self.debug > 2: - inferred = concat + offset_cd - print("given, inferred, offset", offset_cd, inferred, concat) - # self.start_dir: Position of start of central directory - self.start_dir = offset_cd + concat if self.start_dir < 0: raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) + size_cd = endrec[_ECD_SIZE] data = fp.read(size_cd) fp = io.BytesIO(data) total = 0 diff --git a/Misc/NEWS.d/next/Library/2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst b/Misc/NEWS.d/next/Library/2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst new file mode 100644 index 00000000000000..0c518983770d5d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst @@ -0,0 +1 @@ +Improve Zip file validation false positive rate in :func:`zipfile.is_zipfile`. From 64047f7e926b8be8f75cf0b980808ad5bec5e931 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 07:16:56 +0200 Subject: [PATCH 171/713] [3.14] gh-134360 Add processName attribute to `logging.Formatter` docstring (GH-134371) (GH-134405) Co-authored-by: Sofia Toro --- Lib/logging/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 283a1055182b63..357d127c090482 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -591,6 +591,7 @@ class Formatter(object): %(threadName)s Thread name (if available) %(taskName)s Task name (if available) %(process)d Process ID (if available) + %(processName)s Process name (if available) %(message)s The result of record.getMessage(), computed just as the record is emitted """ From b0bf48a13482e121c955f341fc011e30e3e1afe4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 11:29:42 +0200 Subject: [PATCH 172/713] [3.14] Add documentation for remote debugging with pdb (GH-134260) (#134398) --- Doc/library/pdb.rst | 20 +++++++++++++++++++- Lib/pdb.py | 3 ++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index a0304edddf6478..f4b51664545be5 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -80,7 +80,7 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug You can also invoke :mod:`pdb` from the command line to debug other scripts. For example:: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or @@ -104,6 +104,24 @@ useful than quitting the debugger upon program's exit. .. versionchanged:: 3.7 Added the ``-m`` option. +.. option:: -p, --pid + + Attach to the process with the specified PID. + + .. versionadded:: 3.14 + + +To attach to a running Python process for remote debugging, use the ``-p`` or +``--pid`` option with the target process's PID:: + + python -m pdb -p 1234 + +.. note:: + + Attaching to a process that is blocked in a system call or waiting for I/O + will only work once the next bytecode instruction is executed or when the + process receives a signal. + Typical usage to execute a statement under control of the debugger is:: >>> import pdb diff --git a/Lib/pdb.py b/Lib/pdb.py index f89d104fcddb9a..d9e9bc011dbcca 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -3490,7 +3490,8 @@ def help(): _usage = """\ Debug the Python program given by pyfile. Alternatively, an executable module or package to debug can be specified using -the -m switch. +the -m switch. You can also attach to a running Python process +using the -p option with its PID. Initial commands are read from .pdbrc files in your home directory and in the current directory, if they exist. Commands supplied with From 87d7a19ef0a751925be1f703858831c5e0c4fb57 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 11:36:39 +0200 Subject: [PATCH 173/713] [3.14] gh-133980: use atomic store in `PyObject_GenericSetDict` (GH-133988) (#134354) gh-133980: use atomic store in `PyObject_GenericSetDict` (GH-133988) (cherry picked from commit ec39fd2c20323ee9814a1137b1a0819e92efae4e) Co-authored-by: Kumar Aditya --- Lib/test/test_free_threading/test_dict.py | 16 ++++++++++++++++ Objects/object.c | 8 +++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 476cc3178d843f..5d5d4e226caa40 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -228,6 +228,22 @@ def reader_func(): self.assertEqual(count, 0) + def test_racing_object_get_set_dict(self): + e = Exception() + + def writer(): + for i in range(10000): + e.__dict__ = {1:2} + + def reader(): + for i in range(10000): + e.__dict__ + + t1 = Thread(target=writer) + t2 = Thread(target=reader) + + with threading_helper.start_threads([t1, t2]): + pass if __name__ == "__main__": unittest.main() diff --git a/Objects/object.c b/Objects/object.c index 723b0427e69251..af1aa217f75462 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1931,7 +1931,13 @@ PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) return -1; } Py_BEGIN_CRITICAL_SECTION(obj); - Py_XSETREF(*dictptr, Py_NewRef(value)); + PyObject *olddict = *dictptr; + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, Py_NewRef(value)); +#ifdef Py_GIL_DISABLED + _PyObject_XDecRefDelayed(olddict); +#else + Py_XDECREF(olddict); +#endif Py_END_CRITICAL_SECTION(); return 0; } From 7c3e21540b6f33840e5a756c71dbb8487b66e2d4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 13:38:37 +0200 Subject: [PATCH 174/713] [3.14] gh-132542: Only run test_native_id_after_fork if native_id is supported (GH-134408) (GH-134413) (cherry picked from commit e6dde10a69ce62c3b02b3b4e07559d06a9451fc2) Co-authored-by: Noam Cohen --- Lib/test/test_threading.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index dc7172db04c5b7..a9a354647f56ca 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1353,6 +1353,7 @@ def do_flush(*args, **kwargs): assert_python_ok("-c", script) @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(threading, 'get_native_id'), "test needs threading.get_native_id()") def test_native_id_after_fork(self): script = """if True: import threading From ac27008ccdaed0ca587bacb9dd51bfb7503a0ae0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 15:08:20 +0200 Subject: [PATCH 175/713] [3.14] Fix signature of `_curses.assume_default_colors` in the docs (GH-134409) (#134417) Fix signature of `_curses.assume_default_colors` in the docs (GH-134409) (cherry picked from commit dcfc91e4e552e74a43f5fdf049af7a8fe7a784ee) Co-authored-by: sobolevn --- Doc/library/curses.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 5ec23b61396773..0b13c559295f3c 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -68,7 +68,7 @@ The module :mod:`curses` defines the following exception: The module :mod:`curses` defines the following functions: -.. function:: assume_default_colors(fg, bg) +.. function:: assume_default_colors(fg, bg, /) Allow use of default values for colors on terminals supporting this feature. Use this to support transparency in your application. From f7cdee93f48f871b3200c2704bcd96e1b6f49597 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 15:58:09 +0200 Subject: [PATCH 176/713] [3.14] gh-110631: Fix some incorrect indents in the documentation (GH-129312) (#134419) Co-authored-by: Yuki Kobayashi --- Doc/library/functions.rst | 76 ++++++++++++------------- Doc/library/importlib.resources.abc.rst | 48 ++++++++-------- Doc/library/signal.rst | 4 +- Doc/library/socket.rst | 10 ++-- 4 files changed, 69 insertions(+), 69 deletions(-) diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7e367a0f2b6b25..2ecce3dba5a0b9 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -1154,44 +1154,44 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Return a mapping object representing the current local symbol table, with - variable names as the keys, and their currently bound references as the - values. - - At module scope, as well as when using :func:`exec` or :func:`eval` with - a single namespace, this function returns the same namespace as - :func:`globals`. - - At class scope, it returns the namespace that will be passed to the - metaclass constructor. - - When using ``exec()`` or ``eval()`` with separate local and global - arguments, it returns the local namespace passed in to the function call. - - In all of the above cases, each call to ``locals()`` in a given frame of - execution will return the *same* mapping object. Changes made through - the mapping object returned from ``locals()`` will be visible as assigned, - reassigned, or deleted local variables, and assigning, reassigning, or - deleting local variables will immediately affect the contents of the - returned mapping object. - - In an :term:`optimized scope` (including functions, generators, and - coroutines), each call to ``locals()`` instead returns a fresh dictionary - containing the current bindings of the function's local variables and any - nonlocal cell references. In this case, name binding changes made via the - returned dict are *not* written back to the corresponding local variables - or nonlocal cell references, and assigning, reassigning, or deleting local - variables and nonlocal cell references does *not* affect the contents - of previously returned dictionaries. - - Calling ``locals()`` as part of a comprehension in a function, generator, or - coroutine is equivalent to calling it in the containing scope, except that - the comprehension's initialised iteration variables will be included. In - other scopes, it behaves as if the comprehension were running as a nested - function. - - Calling ``locals()`` as part of a generator expression is equivalent to - calling it in a nested generator function. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. .. versionchanged:: 3.12 The behaviour of ``locals()`` in a comprehension has been updated as diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 7a77466bcbaf27..8253a33f591a0b 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -49,44 +49,44 @@ .. method:: open_resource(resource) :abstractmethod: - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. + Returns an opened, :term:`file-like object` for binary reading + of the *resource*. - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. + If the resource cannot be found, :exc:`FileNotFoundError` is + raised. .. method:: resource_path(resource) :abstractmethod: - Returns the file system path to the *resource*. + Returns the file system path to the *resource*. - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. + If the resource does not concretely exist on the file system, + raise :exc:`FileNotFoundError`. .. method:: is_resource(name) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *name* is considered a resource. + :exc:`FileNotFoundError` is raised if *name* does not exist. .. method:: contents() :abstractmethod: - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. + Returns an :term:`iterable` of strings over the contents of + the package. Do note that it is not required that all names + returned by the iterator be actual resources, e.g. it is + acceptable to return names for which :meth:`is_resource` would + be false. + + Allowing non-resource names to be returned is to allow for + situations where how a package and its resources are stored + are known a priori and the non-resource names would be useful. + For instance, returning subdirectory names is allowed so that + when it is known that the package and resources are stored on + the file system then those subdirectory names can be used + directly. + + The abstract method returns an iterable of no items. .. class:: Traversable diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index c28841dbb8cfc8..b0307d3dea1170 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -211,8 +211,8 @@ The variables defined in the :mod:`signal` module are: .. data:: SIGSTKFLT - Stack fault on coprocessor. The Linux kernel does not raise this signal: it - can only be raised in user space. + Stack fault on coprocessor. The Linux kernel does not raise this signal: it + can only be raised in user space. .. availability:: Linux. diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 8fd5187e3a4a36..75fd637045d7d8 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -362,10 +362,10 @@ Exceptions Constants ^^^^^^^^^ - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. +The AF_* and SOCK_* constants are now :class:`AddressFamily` and +:class:`SocketKind` :class:`.IntEnum` collections. - .. versionadded:: 3.4 +.. versionadded:: 3.4 .. data:: AF_UNIX AF_INET @@ -773,9 +773,9 @@ Constants Constant to optimize CPU locality, to be used in conjunction with :data:`SO_REUSEPORT`. - .. versionadded:: 3.11 + .. versionadded:: 3.11 - .. availability:: Linux >= 3.9 + .. availability:: Linux >= 3.9 .. data:: SO_REUSEPORT_LB From 9be568eda44a0888cd03b248e5d18f5a88fec04b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 16:24:23 +0200 Subject: [PATCH 177/713] [3.14] gh-132983: Add documentation for compression.zstd (GH-133911) (#134425) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-132983: Add documentation for compression.zstd (GH-133911) Add documentation for compression & compression.zstd. 🎉 --------- (cherry picked from commit d862b6de1bcff01229318d7138ddaab755723a43) Co-authored-by: Emma Smith Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Sumana Harihareswara Co-authored-by: Martin Panter --- Doc/library/archiving.rst | 4 +- Doc/library/compression.rst | 18 + Doc/library/compression.zstd.rst | 840 +++++++++++++++++++++++++++++++ 3 files changed, 861 insertions(+), 1 deletion(-) create mode 100644 Doc/library/compression.rst create mode 100644 Doc/library/compression.zstd.rst diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst index c9284949af4972..da0b3f8c3e7693 100644 --- a/Doc/library/archiving.rst +++ b/Doc/library/archiving.rst @@ -5,13 +5,15 @@ Data Compression and Archiving ****************************** The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format +gzip, bzip2, lzma, and zstd algorithms, and the creation of ZIP- and tar-format archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` module. .. toctree:: + compression.rst + compression.zstd.rst zlib.rst gzip.rst bz2.rst diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst new file mode 100644 index 00000000000000..618b4a3c2bd170 --- /dev/null +++ b/Doc/library/compression.rst @@ -0,0 +1,18 @@ +The :mod:`!compression` package +=============================== + +.. versionadded:: 3.14 + +The :mod:`!compression` package contains the canonical compression modules +containing interfaces to several different compression algorithms. Some of +these modules have historically been available as separate modules; those will +continue to be available under their original names for compatibility reasons, +and will not be removed without a deprecation cycle. The use of modules in +:mod:`!compression` is encouraged where practical. + +* :mod:`!compression.bz2` -- Re-exports :mod:`bz2` +* :mod:`!compression.gzip` -- Re-exports :mod:`gzip` +* :mod:`!compression.lzma` -- Re-exports :mod:`lzma` +* :mod:`!compression.zlib` -- Re-exports :mod:`zlib` +* :mod:`compression.zstd` -- Wrapper for the Zstandard compression library + diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst new file mode 100644 index 00000000000000..1e1802155a19ec --- /dev/null +++ b/Doc/library/compression.zstd.rst @@ -0,0 +1,840 @@ +:mod:`!compression.zstd` --- Compression compatible with the Zstandard format +============================================================================= + +.. module:: compression.zstd + :synopsis: Low-level interface to compression and decompression routines in + the zstd library. + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/compression/zstd/__init__.py` + +-------------- + +This module provides classes and functions for compressing and decompressing +data using the Zstandard (or *zstd*) compression algorithm. The +`zstd manual `__ +describes Zstandard as "a fast lossless compression algorithm, targeting +real-time compression scenarios at zlib-level and better compression ratios." +Also included is a file interface that supports reading and writing the +contents of ``.zst`` files created by the :program:`zstd` utility, as well as +raw zstd compressed streams. + +The :mod:`!compression.zstd` module contains: + +* The :func:`.open` function and :class:`ZstdFile` class for reading and + writing compressed files. +* The :class:`ZstdCompressor` and :class:`ZstdDecompressor` classes for + incremental (de)compression. +* The :func:`compress` and :func:`decompress` functions for one-shot + (de)compression. +* The :func:`train_dict` and :func:`finalize_dict` functions and the + :class:`ZstdDict` class to train and manage Zstandard dictionaries. +* The :class:`CompressionParameter`, :class:`DecompressionParameter`, and + :class:`Strategy` classes for setting advanced (de)compression parameters. + + +Exceptions +---------- + +.. exception:: ZstdError + + This exception is raised when an error occurs during compression or + decompression, or while initializing the (de)compressor state. + + +Reading and writing compressed files +------------------------------------ + +.. function:: open(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None, encoding=None, errors=None, newline=None) + + Open a Zstandard-compressed file in binary or text mode, returning a + :term:`file object`. + + The *file* argument can be either a file name (given as a + :class:`str`, :class:`bytes` or :term:`path-like ` + object), in which case the named file is opened, or it can be an existing + file object to read from or write to. + + The mode argument can be either ``'rb'`` for reading (default), ``'wb'`` for + overwriting, ``'ab'`` for appending, or ``'xb'`` for exclusive creation. + These can equivalently be given as ``'r'``, ``'w'``, ``'a'``, and ``'x'`` + respectively. You may also open in text mode with ``'rt'``, ``'wt'``, + ``'at'``, and ``'xt'`` respectively. + + When reading, the *options* argument can be a dictionary providing advanced + decompression parameters; see :class:`DecompressionParameter` for detailed + information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be non-None. + The *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + In binary mode, this function is equivalent to the :class:`ZstdFile` + constructor: ``ZstdFile(file, mode, ...)``. In this case, the + *encoding*, *errors*, and *newline* parameters must not be provided. + + In text mode, a :class:`ZstdFile` object is created, and wrapped in an + :class:`io.TextIOWrapper` instance with the specified encoding, error + handling behavior, and line endings. + + +.. class:: ZstdFile(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None) + + Open a Zstandard-compressed file in binary mode. + + A :class:`ZstdFile` can wrap an already-open :term:`file object`, or operate + directly on a named file. The *file* argument specifies either the file + object to wrap, or the name of the file to open (as a :class:`str`, + :class:`bytes` or :term:`path-like ` object). If + wrapping an existing file object, the wrapped file will not be closed when + the :class:`ZstdFile` is closed. + + The *mode* argument can be either ``'rb'`` for reading (default), ``'wb'`` + for overwriting, ``'xb'`` for exclusive creation, or ``'ab'`` for appending. + These can equivalently be given as ``'r'``, ``'w'``, ``'x'`` and ``'a'`` + respectively. + + If *file* is a file object (rather than an actual file name), a mode of + ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. + + When reading, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`DecompressionParameter` for detailed information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be passed. The + *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + :class:`!ZstdFile` supports all the members specified by + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. + Iteration and the :keyword:`with` statement are supported. + + The following method and attributes are also provided: + + .. method:: peek(size=-1) + + Return buffered data without advancing the file position. At least one + byte of data will be returned, unless EOF has been reached. The exact + number of bytes returned is unspecified (the *size* argument is ignored). + + .. note:: While calling :meth:`peek` does not change the file position of + the :class:`ZstdFile`, it may change the position of the underlying + file object (for example, if the :class:`ZstdFile` was constructed by + passing a file object for *file*). + + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. attribute:: name + + The name of the Zstandard file. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + +Compressing and decompressing data in memory +-------------------------------------------- + +.. function:: compress(data, level=None, options=None, zstd_dict=None) + + Compress *data* (a :term:`bytes-like object`), returning the compressed + data as a :class:`bytes` object. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + +.. function:: decompress(data, zstd_dict=None, options=None) + + Decompress *data* (a :term:`bytes-like object`), returning the uncompressed + data as a :class:`bytes` object. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + If *data* is the concatenation of multiple distinct compressed frames, + decompress all of these frames, and return the concatenation of the results. + + +.. class:: ZstdCompressor(level=None, options=None, zstd_dict=None) + + Create a compressor object, which can be used to compress data + incrementally. + + For a more convenient way of compressing a single chunk of data, see the + module-level function :func:`compress`. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an optional instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + + .. method:: compress(data, mode=ZstdCompressor.CONTINUE) + + Compress *data* (a :term:`bytes-like object`), returning a :class:`bytes` + object with compressed data if possible, or otherwise an empty + :class:`!bytes` object. Some of *data* may be buffered internally, for + use in later calls to :meth:`!compress` and :meth:`~.flush`. The returned + data should be concatenated with the output of any previous calls to + :meth:`~.compress`. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, + or :attr:`~.FLUSH_FRAME`. + + When all data has been provided to the compressor, call the + :meth:`~.flush` method to finish the compression process. If + :meth:`~.compress` is called with *mode* set to :attr:`~.FLUSH_FRAME`, + :meth:`~.flush` should not be called, as it would write out a new empty + frame. + + .. method:: flush(mode=ZstdCompressor.FLUSH_FRAME) + + Finish the compression process, returning a :class:`bytes` object + containing any data stored in the compressor's internal buffers. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.FLUSH_BLOCK`, or :attr:`~.FLUSH_FRAME`. + + .. attribute:: CONTINUE + + Collect more data for compression, which may or may not generate output + immediately. This mode optimizes the compression ratio by maximizing the + amount of data per block and frame. + + .. attribute:: FLUSH_BLOCK + + Complete and write a block to the data stream. The data returned so far + can be immediately decompressed. Past data can still be referenced in + future blocks generated by calls to :meth:`~.compress`, + improving compression. + + .. attribute:: FLUSH_FRAME + + Complete and write out a frame. Future data provided to + :meth:`~.compress` will be written into a new frame and + *cannot* reference past data. + + +.. class:: ZstdDecompressor(zstd_dict=None, options=None) + + Create a decompressor object, which can be used to decompress data + incrementally. + + For a more convenient way of decompressing an entire compressed stream at + once, see the module-level function :func:`decompress`. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + .. note:: + This class does not transparently handle inputs containing multiple + compressed frames, unlike the :func:`decompress` function and + :class:`ZstdFile` class. To decompress a multi-frame input, you should + use :func:`decompress`, :class:`ZstdFile` if working with a + :term:`file object`, or multiple :class:`!ZstdDecompressor` instances. + + .. method:: decompress(data, max_length=-1) + + Decompress *data* (a :term:`bytes-like object`), returning + uncompressed data as bytes. Some of *data* may be buffered + internally, for use in later calls to :meth:`!decompress`. + The returned data should be concatenated with the output of any previous + calls to :meth:`!decompress`. + + If *max_length* is non-negative, the method returns at most *max_length* + bytes of decompressed data. If this limit is reached and further + output can be produced, the :attr:`~.needs_input` attribute will + be set to ``False``. In this case, the next call to + :meth:`~.decompress` may provide *data* as ``b''`` to obtain + more of the output. + + If all of the input data was decompressed and returned (either + because this was less than *max_length* bytes, or because + *max_length* was negative), the :attr:`~.needs_input` attribute + will be set to ``True``. + + Attempting to decompress data after the end of a frame will raise a + :exc:`ZstdError`. Any data found after the end of the frame is ignored + and saved in the :attr:`~.unused_data` attribute. + + .. attribute:: eof + + ``True`` if the end-of-stream marker has been reached. + + .. attribute:: unused_data + + Data found after the end of the compressed stream. + + Before the end of the stream is reached, this will be ``b''``. + + .. attribute:: needs_input + + ``False`` if the :meth:`.decompress` method can provide more + decompressed data before requiring new compressed input. + + +Zstandard dictionaries +---------------------- + + +.. function:: train_dict(samples, dict_size) + + Train a Zstandard dictionary, returning a :class:`ZstdDict` instance. + Zstandard dictionaries enable more efficient compression of smaller sizes + of data, which is traditionally difficult to compress due to less + repetition. If you are compressing multiple similar groups of data (such as + similar files), Zstandard dictionaries can improve compression ratios and + speed significantly. + + The *samples* argument (an iterable of :class:`bytes` objects), is the + population of samples used to train the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. The Zstandard documentation suggests an + absolute maximum of no more than 100 KB, but the maximum can often be smaller + depending on the data. Larger dictionaries generally slow down compression, + but improve compression ratios. Smaller dictionaries lead to faster + compression, but reduce the compression ratio. + + +.. function:: finalize_dict(zstd_dict, /, samples, dict_size, level) + + An advanced function for converting a "raw content" Zstandard dictionary into + a regular Zstandard dictionary. "Raw content" dictionaries are a sequence of + bytes that do not need to follow the structure of a normal Zstandard + dictionary. + + The *zstd_dict* argument is a :class:`ZstdDict` instance with + the :attr:`~ZstdDict.dict_content` containing the raw dictionary contents. + + The *samples* argument (an iterable of :class:`bytes` objects), contains + sample data for generating the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. See :func:`train_dict` for + suggestions on the maximum dictionary size. + + The *level* argument (an integer) is the compression level expected to be + passed to the compressors using this dictionary. The dictionary information + varies for each compression level, so tuning for the proper compression + level can make compression more efficient. + + +.. class:: ZstdDict(dict_content, /, *, is_raw=False) + + A wrapper around Zstandard dictionaries. Dictionaries can be used to improve + the compression of many small chunks of data. Use :func:`train_dict` if you + need to train a new dictionary from sample data. + + The *dict_content* argument (a :term:`bytes-like object`), is the already + trained dictionary information. + + The *is_raw* argument, a boolean, is an advanced parameter controlling the + meaning of *dict_content*. ``True`` means *dict_content* is a "raw content" + dictionary, without any format restrictions. ``False`` means *dict_content* + is an ordinary Zstandard dictionary, created from Zstandard functions, + for example, :func:`train_dict` or the external :program:`zstd` CLI. + + When passing a :class:`!ZstdDict` to a function, the + :attr:`!as_digested_dict` and :attr:`!as_undigested_dict` attributes can + control how the dictionary is loaded by passing them as the ``zstd_dict`` + argument, for example, ``compress(data, zstd_dict=zd.as_digested_dict)``. + Digesting a dictionary is a costly operation that occurs when loading a + Zstandard dictionary. When making multiple calls to compression or + decompression, passing a digested dictionary will reduce the overhead of + loading the dictionary. + + .. list-table:: Difference for compression + :widths: 10 14 10 + :header-rows: 1 + + * - + - Digested dictionary + - Undigested dictionary + * - Advanced parameters of the compressor which may be overridden by + the dictionary's parameters + - ``window_log``, ``hash_log``, ``chain_log``, ``search_log``, + ``min_match``, ``target_length``, ``strategy``, + ``enable_long_distance_matching``, ``ldm_hash_log``, + ``ldm_min_match``, ``ldm_bucket_size_log``, ``ldm_hash_rate_log``, + and some non-public parameters. + - None + * - :class:`!ZstdDict` internally caches the dictionary + - Yes. It's faster when loading a digested dictionary again with the + same compression level. + - No. If you wish to load an undigested dictionary multiple times, + consider reusing a compressor object. + + If passing a :class:`!ZstdDict` without any attribute, an undigested + dictionary is passed by default when compressing and a digested dictionary + is generated if necessary and passed by default when decompressing. + + .. attribute:: dict_content + + The content of the Zstandard dictionary, a ``bytes`` object. It's the + same as the *dict_content* argument in the ``__init__`` method. It can + be used with other programs, such as the ``zstd`` CLI program. + + .. attribute:: dict_id + + Identifier of the Zstandard dictionary, a non-negative int value. + + Non-zero means the dictionary is ordinary, created by Zstandard + functions and following the Zstandard format. + + ``0`` means a "raw content" dictionary, free of any format restriction, + used for advanced users. + + .. note:: + + The meaning of ``0`` for :attr:`!ZstdDict.dict_id` is different + from the ``dictionary_id`` attribute to the :func:`get_frame_info` + function. + + .. attribute:: as_digested_dict + + Load as a digested dictionary. + + .. attribute:: as_undigested_dict + + Load as an undigested dictionary. + + +Advanced parameter control +-------------------------- + +.. class:: CompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced compression parameter + keys that can be used when compressing data. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Parameters are optional; any omitted parameter will have it's value selected + automatically. + + Example getting the lower and upper bound of :attr:`~.compression_level`:: + + lower, upper = CompressionParameter.compression_level.bounds() + + Example setting the :attr:`~.window_log` to the maximum size:: + + _lower, upper = CompressionParameter.window_log.bounds() + options = {CompressionParameter.window_log: upper} + compress(b'venezuelan beaver cheese', options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a compression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. For example, to get the valid values for + :attr:`~.compression_level`, one may check the result of + ``CompressionParameter.compression_level.bounds()``. + + Both the lower and upper bounds are inclusive. + + .. attribute:: compression_level + + A high-level means of setting other compression parameters that affect + the speed and ratio of compressing data. Setting the level to zero uses + :attr:`COMPRESSION_LEVEL_DEFAULT`. + + .. attribute:: window_log + + Maximum allowed back-reference distance the compressor can use when + compressing data, expressed as power of two, ``1 << window_log`` bytes. + This parameter greatly influences the memory usage of compression. Higher + values require more memory but gain better compression values. + + A value of zero causes the value to be selected automatically. + + .. attribute:: hash_log + + Size of the initial probe table, as a power of two. The resulting memory + usage is ``1 << (hash_log+2)`` bytes. Larger tables improve compression + ratio of strategies <= :attr:`~Strategy.dfast`, and improve compression + speed of strategies > :attr:`~Strategy.dfast`. + + A value of zero causes the value to be selected automatically. + + .. attribute:: chain_log + + Size of the multi-probe search table, as a power of two. The resulting + memory usage is ``1 << (chain_log+2)`` bytes. Larger tables result in + better and slower compression. This parameter has no effect for the + :attr:`~Strategy.fast` strategy. It's still useful when using + :attr:`~Strategy.dfast` strategy, in which case it defines a secondary + probe table. + + A value of zero causes the value to be selected automatically. + + .. attribute:: search_log + + Number of search attempts, as a power of two. More attempts result in + better and slower compression. This parameter is useless for + :attr:`~Strategy.fast` and :attr:`~Strategy.dfast` strategies. + + A value of zero causes the value to be selected automatically. + + .. attribute:: min_match + + Minimum size of searched matches. Larger values increase compression and + decompression speed, but decrease ratio. Note that Zstandard can still + find matches of smaller size, it just tweaks its search algorithm to look + for this size and larger. For all strategies < :attr:`~Strategy.btopt`, + the effective minimum is ``4``; for all strategies + > :attr:`~Strategy.fast`, the effective maximum is ``6``. + + A value of zero causes the value to be selected automatically. + + .. attribute:: target_length + + The impact of this field depends on the selected :class:`Strategy`. + + For strategies :attr:`~Strategy.btopt`, :attr:`~Strategy.btultra` and + :attr:`~Strategy.btultra2`, the value is the length of a match + considered "good enough" to stop searching. Larger values make + compression ratios better, but compresses slower. + + For strategy :attr:`~Strategy.fast`, it is the distance between match + sampling. Larger values make compression faster, but with a worse + compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: strategy + + The higher the value of selected strategy, the more complex the + compression technique used by zstd, resulting in higher compression + ratios but slower compression. + + .. seealso:: :class:`Strategy` + + .. attribute:: enable_long_distance_matching + + Long distance matching can be used to improve compression for large + inputs by finding large matches at greater distances. It increases memory + usage and window size. + + ``True`` or ``1`` enable long distance matching while ``False`` or ``0`` + disable it. + + Enabling this parameter increases default + :attr:`~CompressionParameter.window_log` to 128 MiB except when expressly + set to a different value. This setting is enabled by default if + :attr:`!window_log` >= 128 MiB and the compression + strategy >= :attr:`~Strategy.btopt` (compression level 16+). + + .. attribute:: ldm_hash_log + + Size of the table for long distance matching, as a power of two. Larger + values increase memory usage and compression ratio, but decrease + compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_min_match + + Minimum match size for long distance matcher. Larger or too small values + can often decrease the compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_bucket_size_log + + Log size of each bucket in the long distance matcher hash table for + collision resolution. Larger values improve collision resolution but + decrease compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_hash_rate_log + + Frequency of inserting/looking up entries into the long distance matcher + hash table. Larger values improve compression speed. Deviating far from + the default value will likely result in a compression ratio decrease. + + A value of zero causes the value to be selected automatically. + + .. attribute:: checksum_flag + + A four-byte checksum using XXHash64 of the uncompressed content is + written at the end of each frame. Zstandard's decompression code verifies + the checksum. If there is a mismatch a :class:`ZstdError` exception is + raised. + + ``True`` or ``1`` enable checksum generation while ``False`` or ``0`` + disable it. + + .. attribute:: dict_id_flag + + When compressing with a :class:`ZstdDict`, the dictionary's ID is written + into the frame header. + + ``True`` or ``1`` enable storing the dictionary ID while ``False`` or + ``0`` disable it. + + .. attribute:: nb_workers + + Select how many threads will be spawned to compress in parallel. When + :attr:`!nb_workers` > 0, enables multi-threaded compression, a value of + ``1`` means "one-thread multi-threaded mode". More workers improve speed, + but also increase memory usage and slightly reduce compression ratio. + + A value of zero disables multi-threading. + + .. attribute:: job_size + + Size of a compression job, in bytes. This value is enforced only when + :attr:`~CompressionParameter.nb_workers` >= 1. Each compression job is + completed in parallel, so this value can indirectly impact the number of + active threads. + + A value of zero causes the value to be selected automatically. + + .. attribute:: overlap_log + + Sets how much data is reloaded from previous jobs (threads) for new jobs + to be used by the look behind window during compression. This value is + only used when :attr:`~CompressionParameter.nb_workers` >= 1. Acceptable + values vary from 0 to 9. + + * 0 means dynamically set the overlap amount + * 1 means no overlap + * 9 means use a full window size from the previous job + + Each increment halves/doubles the overlap size. "8" means an overlap of + ``window_size/2``, "7" means an overlap of ``window_size/4``, etc. + +.. class:: DecompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced decompression parameter + keys that can be used when decompressing data. Parameters are optional; any + omitted parameter will have it's value selected automatically. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Example setting the :attr:`~.window_log_max` to the maximum size:: + + data = compress(b'Some very long buffer of bytes...') + + _lower, upper = DecompressionParameter.window_log_max.bounds() + + options = {DecompressionParameter.window_log_max: upper} + decompress(data, options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a decompression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. + + Both the lower and upper bounds are inclusive. + + .. attribute:: window_log_max + + The base-two logarithm of the maximum size of the window used during + decompression. This can be useful to limit the amount of memory used when + decompressing data. A larger maximum window size leads to faster + decompression. + + A value of zero causes the value to be selected automatically. + + +.. class:: Strategy() + + An :class:`~enum.IntEnum` containing strategies for compression. + Higher-numbered strategies correspond to more complex and slower + compression. + + .. note:: + + The values of attributes of :class:`!Strategy` are not necessarily stable + across zstd versions. Only the ordering of the attributes may be relied + upon. The attributes are listed below in order. + + The following strategies are available: + + .. attribute:: fast + + .. attribute:: dfast + + .. attribute:: greedy + + .. attribute:: lazy + + .. attribute:: lazy2 + + .. attribute:: btlazy2 + + .. attribute:: btopt + + .. attribute:: btultra + + .. attribute:: btultra2 + + +Miscellaneous +------------- + +.. function:: get_frame_info(frame_buffer) + + Retrieve a :class:`FrameInfo` object containing metadata about a Zstandard + frame. Frames contain metadata related to the compressed data they hold. + + +.. class:: FrameInfo + + Metadata related to a Zstandard frame. + + .. attribute:: decompressed_size + + The size of the decompressed contents of the frame. + + .. attribute:: dictionary_id + + An integer representing the Zstandard dictionary ID needed for + decompressing the frame. ``0`` means the dictionary ID was not + recorded in the frame header. This may mean that a Zstandard dictionary + is not needed, or that the ID of a required dictionary was not recorded. + + +.. attribute:: COMPRESSION_LEVEL_DEFAULT + + The default compression level for Zstandard: ``3``. + + +.. attribute:: zstd_version_info + + Version number of the runtime zstd library as a tuple of integers + (major, minor, release). + + +Examples +-------- + +Reading in a compressed file: + +.. code-block:: python + + from compression import zstd + + with zstd.open("file.zst") as f: + file_content = f.read() + +Creating a compressed file: + +.. code-block:: python + + from compression import zstd + + data = b"Insert Data Here" + with zstd.open("file.zst", "w") as f: + f.write(data) + +Compressing data in memory: + +.. code-block:: python + + from compression import zstd + + data_in = b"Insert Data Here" + data_out = zstd.compress(data_in) + +Incremental compression: + +.. code-block:: python + + from compression import zstd + + comp = zstd.ZstdCompressor() + out1 = comp.compress(b"Some data\n") + out2 = comp.compress(b"Another piece of data\n") + out3 = comp.compress(b"Even more data\n") + out4 = comp.flush() + # Concatenate all the partial results: + result = b"".join([out1, out2, out3, out4]) + +Writing compressed data to an already-open file: + +.. code-block:: python + + from compression import zstd + + with open("myfile", "wb") as f: + f.write(b"This data will not be compressed\n") + with zstd.open(f, "w") as zstf: + zstf.write(b"This *will* be compressed\n") + f.write(b"Not compressed\n") + +Creating a compressed file using compression parameters: + +.. code-block:: python + + from compression import zstd + + options = { + zstd.CompressionParameter.checksum_flag: 1 + } + with zstd.open("file.zst", "w", options=options) as f: + f.write(b"Mind if I squeeze in?") From f1a9d891071bd264237211ec5251dfc82bc29fb3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 16:27:13 +0200 Subject: [PATCH 178/713] [3.14] gh-90871: fix connection backlog offset in asyncio (gh-134392) (gh-134421) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 109f7597d29f0b504e9b42cb398b44177157abaa) Co-authored-by: Christian Harries <68507104+ChristianHrs@users.noreply.github.com> Co-authored-by: Łukasz Langa Co-authored-by: Kumar Aditya --- Lib/asyncio/selector_events.py | 2 +- Lib/test/test_asyncio/test_selector_events.py | 16 ++++++++++++++-- ...2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst | 2 ++ 3 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 22147451fa7ebd..6ad84044adf146 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,7 +173,7 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index de81936b7456f2..aab6a779170eb9 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -347,6 +347,18 @@ def test_process_events_write_cancelled(self): selectors.EVENT_WRITE)]) self.loop._remove_writer.assert_called_with(1) + def test_accept_connection_zero_one(self): + for backlog in [0, 1]: + sock = mock.Mock() + sock.accept.return_value = (mock.Mock(), mock.Mock()) + with self.subTest(backlog): + mock_obj = mock.patch.object + with mock_obj(self.loop, '_accept_connection2') as accept2_mock: + self.loop._accept_connection( + mock.Mock(), sock, backlog=backlog) + self.loop.run_until_complete(asyncio.sleep(0)) + self.assertEqual(sock.accept.call_count, backlog + 1) + def test_accept_connection_multiple(self): sock = mock.Mock() sock.accept.return_value = (mock.Mock(), mock.Mock()) @@ -362,7 +374,7 @@ def test_accept_connection_multiple(self): self.loop._accept_connection( mock.Mock(), sock, backlog=backlog) self.loop.run_until_complete(asyncio.sleep(0)) - self.assertEqual(sock.accept.call_count, backlog) + self.assertEqual(sock.accept.call_count, backlog + 1) def test_accept_connection_skip_connectionabortederror(self): sock = mock.Mock() @@ -388,7 +400,7 @@ def mock_sock_accept(): # as in test_accept_connection_multiple avoid task pending # warnings by using asyncio.sleep(0) self.loop.run_until_complete(asyncio.sleep(0)) - self.assertEqual(sock.accept.call_count, backlog) + self.assertEqual(sock.accept.call_count, backlog + 1) class SelectorTransportTests(test_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst b/Misc/NEWS.d/next/Library/2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst new file mode 100644 index 00000000000000..49397c9705ecfe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst @@ -0,0 +1,2 @@ +Fixed an off by one error concerning the backlog parameter in +:meth:`~asyncio.loop.create_unix_server`. Contributed by Christian Harries. From cd3395a8b10f6bab912670e1d47d432aa2a21e71 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 16:33:35 +0200 Subject: [PATCH 179/713] [3.14] gh-132246: Add special buffer methods to C API Type Object docs (gh-132247) (gh-134426) Two special methods, __buffer__ and __release_buffer__ were added to Python 3.12 by PEP 688. The C API Type Object documentation for slots includes `tp_as_buffer`, and sub-slots `bf_getbuffer`, `bf_releasebuffer` but does not refer to the Python Data Model version of those. Add the missing references. (cherry picked from commit b529b60fc239d19245e5fafd0514d90097c2eb40) Co-authored-by: Cody Maloney --- Doc/c-api/typeobj.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 64353a8daca5f6..5df0c0fe608e53 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -79,7 +79,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | | | | __delattr__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | :ref:`sub-slots` | | | | % | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -325,9 +325,10 @@ sub-slots +---------------------------------------------------------+-----------------------------------+---------------+ | | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | __buffer__ | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | __release_\ | + | | | buffer\__ | +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: From 2ffc10bd3988c244b658f7e5c97807a938264664 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 16:47:56 +0200 Subject: [PATCH 180/713] [3.14] gh-132775: Support Fallbacks in _PyObject_GetXIData() (gh-134418) It now supports a "full" fallback to _PyFunction_GetXIData() and then `_PyPickle_GetXIData()`. There's also room for other fallback modes if that later makes sense. (cherry picked from commit 88f8102a8f, AKA gh-133482) Co-authored-by: Eric Snow --- Doc/data/python3.14.abi | 62861 ++++++++-------- Include/internal/pycore_crossinterp.h | 25 +- .../pycore_crossinterp_data_registry.h | 4 +- Lib/test/support/import_helper.py | 2 +- Lib/test/test_crossinterp.py | 511 +- Modules/_interpchannelsmodule.c | 8 +- Modules/_interpqueuesmodule.c | 4 +- Modules/_interpreters_common.h | 4 +- Modules/_interpretersmodule.c | 2 +- Modules/_testinternalcapi.c | 9 +- Python/crossinterp.c | 89 +- Python/crossinterp_data_lookup.h | 62 +- 12 files changed, 32041 insertions(+), 31540 deletions(-) diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index c9959bef98fd07..decb2ce3c6f39e 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -1,31401 +1,31460 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 19c55dd65983d7..45fa47d62c78a3 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -131,7 +131,23 @@ PyAPI_FUNC(void) _PyXIData_Clear(PyInterpreterState *, _PyXIData_t *); /* getting cross-interpreter data */ -typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *); +typedef int xidata_fallback_t; +#define _PyXIDATA_XIDATA_ONLY (0) +#define _PyXIDATA_FULL_FALLBACK (1) + +// Technically, we don't need two different function types; +// we could go with just the fallback one. However, only container +// types like tuple need it, so always having the extra arg would be +// a bit unfortunate. It's also nice to be able to clearly distinguish +// between types that might call _PyObject_GetXIData() and those that won't. +// +typedef int (*xidatafunc)(PyThreadState *, PyObject *, _PyXIData_t *); +typedef int (*xidatafbfunc)( + PyThreadState *, PyObject *, xidata_fallback_t, _PyXIData_t *); +typedef struct { + xidatafunc basic; + xidatafbfunc fallback; +} _PyXIData_getdata_t; PyAPI_FUNC(PyObject *) _PyXIData_GetNotShareableErrorType(PyThreadState *); PyAPI_FUNC(void) _PyXIData_SetNotShareableError(PyThreadState *, const char *); @@ -140,16 +156,21 @@ PyAPI_FUNC(void) _PyXIData_FormatNotShareableError( const char *, ...); -PyAPI_FUNC(xidatafunc) _PyXIData_Lookup( +PyAPI_FUNC(_PyXIData_getdata_t) _PyXIData_Lookup( PyThreadState *, PyObject *); PyAPI_FUNC(int) _PyObject_CheckXIData( PyThreadState *, PyObject *); +PyAPI_FUNC(int) _PyObject_GetXIDataNoFallback( + PyThreadState *, + PyObject *, + _PyXIData_t *); PyAPI_FUNC(int) _PyObject_GetXIData( PyThreadState *, PyObject *, + xidata_fallback_t, _PyXIData_t *); // _PyObject_GetXIData() for bytes diff --git a/Include/internal/pycore_crossinterp_data_registry.h b/Include/internal/pycore_crossinterp_data_registry.h index 8f4bcb948e5a45..fbb4cad5cac32e 100644 --- a/Include/internal/pycore_crossinterp_data_registry.h +++ b/Include/internal/pycore_crossinterp_data_registry.h @@ -17,7 +17,7 @@ typedef struct _xid_regitem { /* This is NULL for builtin types. */ PyObject *weakref; size_t refcount; - xidatafunc getdata; + _PyXIData_getdata_t getdata; } _PyXIData_regitem_t; typedef struct { @@ -30,7 +30,7 @@ typedef struct { PyAPI_FUNC(int) _PyXIData_RegisterClass( PyThreadState *, PyTypeObject *, - xidatafunc); + _PyXIData_getdata_t); PyAPI_FUNC(int) _PyXIData_UnregisterClass( PyThreadState *, PyTypeObject *); diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index edb734d294f287..0af63501f93bc8 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -438,5 +438,5 @@ def ensure_module_imported(name, *, clearnone=True): if sys.modules.get(name) is not None: mod = sys.modules[name] else: - mod, _, _ = _force_import(name, False, True, clearnone) + mod, _, _ = _ensure_module(name, False, True, clearnone) return mod diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index cddacbc9970052..c54635eaeab3f9 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -5,6 +5,7 @@ import sys import types import unittest +import warnings from test.support import import_helper @@ -16,13 +17,281 @@ from test import _crossinterp_definitions as defs -BUILTIN_TYPES = [o for _, o in __builtins__.items() - if isinstance(o, type)] -EXCEPTION_TYPES = [cls for cls in BUILTIN_TYPES +@contextlib.contextmanager +def ignore_byteswarning(): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=BytesWarning) + yield + + +# builtin types + +BUILTINS_TYPES = [o for _, o in __builtins__.items() if isinstance(o, type)] +EXCEPTION_TYPES = [cls for cls in BUILTINS_TYPES if issubclass(cls, BaseException)] OTHER_TYPES = [o for n, o in vars(types).items() if (isinstance(o, type) and - n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] + n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] +BUILTIN_TYPES = [ + *BUILTINS_TYPES, + *OTHER_TYPES, +] + +# builtin exceptions + +try: + raise Exception +except Exception as exc: + CAUGHT = exc +EXCEPTIONS_WITH_SPECIAL_SIG = { + BaseExceptionGroup: (lambda msg: (msg, [CAUGHT])), + ExceptionGroup: (lambda msg: (msg, [CAUGHT])), + UnicodeError: (lambda msg: (None, msg, None, None, None)), + UnicodeEncodeError: (lambda msg: ('utf-8', '', 1, 3, msg)), + UnicodeDecodeError: (lambda msg: ('utf-8', b'', 1, 3, msg)), + UnicodeTranslateError: (lambda msg: ('', 1, 3, msg)), +} +BUILTIN_EXCEPTIONS = [ + *(cls(*sig('error!')) for cls, sig in EXCEPTIONS_WITH_SPECIAL_SIG.items()), + *(cls('error!') for cls in EXCEPTION_TYPES + if cls not in EXCEPTIONS_WITH_SPECIAL_SIG), +] + +# other builtin objects + +METHOD = defs.SpamOkay().okay +BUILTIN_METHOD = [].append +METHOD_DESCRIPTOR_WRAPPER = str.join +METHOD_WRAPPER = object().__str__ +WRAPPER_DESCRIPTOR = object.__init__ +BUILTIN_WRAPPERS = { + METHOD: types.MethodType, + BUILTIN_METHOD: types.BuiltinMethodType, + dict.__dict__['fromkeys']: types.ClassMethodDescriptorType, + types.FunctionType.__code__: types.GetSetDescriptorType, + types.FunctionType.__globals__: types.MemberDescriptorType, + METHOD_DESCRIPTOR_WRAPPER: types.MethodDescriptorType, + METHOD_WRAPPER: types.MethodWrapperType, + WRAPPER_DESCRIPTOR: types.WrapperDescriptorType, + staticmethod(defs.SpamOkay.okay): None, + classmethod(defs.SpamOkay.okay): None, + property(defs.SpamOkay.okay): None, +} +BUILTIN_FUNCTIONS = [ + # types.BuiltinFunctionType + len, + sys.is_finalizing, + sys.exit, + _testinternalcapi.get_crossinterp_data, +] +assert 'emptymod' not in sys.modules +with import_helper.ready_to_import('emptymod', ''): + import emptymod as EMPTYMOD +MODULES = [ + sys, + defs, + unittest, + EMPTYMOD, +] +OBJECT = object() +EXCEPTION = Exception() +LAMBDA = (lambda: None) +BUILTIN_SIMPLE = [ + OBJECT, + # singletons + None, + True, + False, + Ellipsis, + NotImplemented, + # bytes + *(i.to_bytes(2, 'little', signed=True) + for i in range(-1, 258)), + # str + 'hello world', + '你好世界', + '', + # int + sys.maxsize + 1, + sys.maxsize, + -sys.maxsize - 1, + -sys.maxsize - 2, + *range(-1, 258), + 2**1000, + # float + 0.0, + 1.1, + -1.0, + 0.12345678, + -0.12345678, +] +TUPLE_EXCEPTION = (0, 1.0, EXCEPTION) +TUPLE_OBJECT = (0, 1.0, OBJECT) +TUPLE_NESTED_EXCEPTION = (0, 1.0, (EXCEPTION,)) +TUPLE_NESTED_OBJECT = (0, 1.0, (OBJECT,)) +MEMORYVIEW_EMPTY = memoryview(b'') +MEMORYVIEW_NOT_EMPTY = memoryview(b'spam'*42) +MAPPING_PROXY_EMPTY = types.MappingProxyType({}) +BUILTIN_CONTAINERS = [ + # tuple (flat) + (), + (1,), + ("hello", "world", ), + (1, True, "hello"), + TUPLE_EXCEPTION, + TUPLE_OBJECT, + # tuple (nested) + ((1,),), + ((1, 2), (3, 4)), + ((1, 2), (3, 4), (5, 6)), + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, + # buffer + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + # list + [], + [1, 2, 3], + [[1], (2,), {3: 4}], + # dict + {}, + {1: 7, 2: 8, 3: 9}, + {1: [1], 2: (2,), 3: {3: 4}}, + # set + set(), + {1, 2, 3}, + {frozenset({1}), (2,)}, + # frozenset + frozenset([]), + frozenset({frozenset({1}), (2,)}), + # bytearray + bytearray(b''), + # other + MAPPING_PROXY_EMPTY, + types.SimpleNamespace(), +] +ns = {} +exec(""" +try: + raise Exception +except Exception as exc: + TRACEBACK = exc.__traceback__ + FRAME = TRACEBACK.tb_frame +""", ns, ns) +BUILTIN_OTHER = [ + # types.CellType + types.CellType(), + # types.FrameType + ns['FRAME'], + # types.TracebackType + ns['TRACEBACK'], +] +del ns + +# user-defined objects + +USER_TOP_INSTANCES = [c(*a) for c, a in defs.TOP_CLASSES.items()] +USER_NESTED_INSTANCES = [c(*a) for c, a in defs.NESTED_CLASSES.items()] +USER_INSTANCES = [ + *USER_TOP_INSTANCES, + *USER_NESTED_INSTANCES, +] +USER_EXCEPTIONS = [ + defs.MimimalError('error!'), +] + +# shareable objects + +TUPLES_WITHOUT_EQUALITY = [ + TUPLE_EXCEPTION, + TUPLE_OBJECT, + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, +] +_UNSHAREABLE_SIMPLE = [ + Ellipsis, + NotImplemented, + OBJECT, + sys.maxsize + 1, + -sys.maxsize - 2, + 2**1000, +] +with ignore_byteswarning(): + _SHAREABLE_SIMPLE = [o for o in BUILTIN_SIMPLE + if o not in _UNSHAREABLE_SIMPLE] + _SHAREABLE_CONTAINERS = [ + *(o for o in BUILTIN_CONTAINERS if type(o) is memoryview), + *(o for o in BUILTIN_CONTAINERS + if type(o) is tuple and o not in TUPLES_WITHOUT_EQUALITY), + ] + _UNSHAREABLE_CONTAINERS = [o for o in BUILTIN_CONTAINERS + if o not in _SHAREABLE_CONTAINERS] +SHAREABLE = [ + *_SHAREABLE_SIMPLE, + *_SHAREABLE_CONTAINERS, +] +NOT_SHAREABLE = [ + *_UNSHAREABLE_SIMPLE, + *_UNSHAREABLE_CONTAINERS, + *BUILTIN_TYPES, + *BUILTIN_WRAPPERS, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *MODULES, + *BUILTIN_OTHER, + # types.CodeType + *(f.__code__ for f in defs.FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + # types.FunctionType + *defs.FUNCTIONS, + defs.SpamOkay.okay, + LAMBDA, + *defs.FUNCTION_LIKE, + # coroutines and generators + *defs.FUNCTION_LIKE_APPLIED, + # user classes + *defs.CLASSES, + *USER_INSTANCES, + # user exceptions + *USER_EXCEPTIONS, +] + +# pickleable objects + +PICKLEABLE = [ + *BUILTIN_SIMPLE, + *(o for o in BUILTIN_CONTAINERS if o not in [ + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + MAPPING_PROXY_EMPTY, + ] or type(o) is dict), + *BUILTINS_TYPES, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *defs.TOP_FUNCTIONS, + defs.SpamOkay.okay, + *defs.FUNCTION_LIKE, + *defs.TOP_CLASSES, + *USER_TOP_INSTANCES, + *USER_EXCEPTIONS, + # from OTHER_TYPES + types.NoneType, + types.EllipsisType, + types.NotImplementedType, + types.GenericAlias, + types.UnionType, + types.SimpleNamespace, + # from BUILTIN_WRAPPERS + METHOD, + BUILTIN_METHOD, + METHOD_DESCRIPTOR_WRAPPER, + METHOD_WRAPPER, + WRAPPER_DESCRIPTOR, +] +assert not any(isinstance(o, types.MappingProxyType) for o in PICKLEABLE) + + +# helpers DEFS = defs with open(code_defs.__file__) as infile: @@ -111,6 +380,77 @@ class _GetXIDataTests(unittest.TestCase): MODE = None + def assert_functions_equal(self, func1, func2): + assert type(func1) is types.FunctionType, repr(func1) + assert type(func2) is types.FunctionType, repr(func2) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__code__, func2.__code__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + # We don't worry about __globals__ for now. + + def assert_exc_args_equal(self, exc1, exc2): + args1 = exc1.args + args2 = exc2.args + if isinstance(exc1, ExceptionGroup): + self.assertIs(type(args1), type(args2)) + self.assertEqual(len(args1), 2) + self.assertEqual(len(args1), len(args2)) + self.assertEqual(args1[0], args2[0]) + group1 = args1[1] + group2 = args2[1] + self.assertEqual(len(group1), len(group2)) + for grouped1, grouped2 in zip(group1, group2): + # Currently the "extra" attrs are not preserved + # (via __reduce__). + self.assertIs(type(exc1), type(exc2)) + self.assert_exc_equal(grouped1, grouped2) + else: + self.assertEqual(args1, args2) + + def assert_exc_equal(self, exc1, exc2): + self.assertIs(type(exc1), type(exc2)) + + if type(exc1).__eq__ is not object.__eq__: + self.assertEqual(exc1, exc2) + + self.assert_exc_args_equal(exc1, exc2) + # XXX For now we do not preserve tracebacks. + if exc1.__traceback__ is not None: + self.assertEqual(exc1.__traceback__, exc2.__traceback__) + self.assertEqual( + getattr(exc1, '__notes__', None), + getattr(exc2, '__notes__', None), + ) + # We assume there are no cycles. + if exc1.__cause__ is None: + self.assertIs(exc1.__cause__, exc2.__cause__) + else: + self.assert_exc_equal(exc1.__cause__, exc2.__cause__) + if exc1.__context__ is None: + self.assertIs(exc1.__context__, exc2.__context__) + else: + self.assert_exc_equal(exc1.__context__, exc2.__context__) + + def assert_equal_or_equalish(self, obj, expected): + cls = type(expected) + if cls.__eq__ is not object.__eq__: + self.assertEqual(obj, expected) + elif cls is types.FunctionType: + self.assert_functions_equal(obj, expected) + elif isinstance(expected, BaseException): + self.assert_exc_equal(obj, expected) + elif cls is types.MethodType: + raise NotImplementedError(cls) + elif cls is types.BuiltinMethodType: + raise NotImplementedError(cls) + elif cls is types.MethodWrapperType: + raise NotImplementedError(cls) + elif cls.__bases__ == (object,): + self.assertEqual(obj.__dict__, expected.__dict__) + else: + raise NotImplementedError(cls) + def get_xidata(self, obj, *, mode=None): mode = self._resolve_mode(mode) return _testinternalcapi.get_crossinterp_data(obj, mode) @@ -126,35 +466,37 @@ def _get_roundtrip(self, obj, mode): def assert_roundtrip_identical(self, values, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIs(got, obj) def assert_roundtrip_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) - self.assertEqual(got, obj) + if got is obj: + continue self.assertIs(type(got), type(obj) if expecttype is None else expecttype) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_equal_not_identical(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), type(obj) if expecttype is None else expecttype) - self.assertEqual(got, obj) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_not_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), @@ -164,7 +506,7 @@ def assert_roundtrip_not_equal(self, values, *, def assert_not_shareable(self, values, exctype=None, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): with self.assertRaises(NotShareableError) as cm: _testinternalcapi.get_crossinterp_data(obj, mode) if exctype is not None: @@ -182,49 +524,26 @@ class PickleTests(_GetXIDataTests): MODE = 'pickle' def test_shareable(self): - self.assert_roundtrip_equal([ - # singletons - None, - True, - False, - # bytes - *(i.to_bytes(2, 'little', signed=True) - for i in range(-1, 258)), - # str - 'hello world', - '你好世界', - '', - # int - sys.maxsize, - -sys.maxsize - 1, - *range(-1, 258), - # float - 0.0, - 1.1, - -1.0, - 0.12345678, - -0.12345678, - # tuple - (), - (1,), - ("hello", "world", ), - (1, True, "hello"), - ((1,),), - ((1, 2), (3, 4)), - ((1, 2), (3, 4), (5, 6)), - ]) - # not shareable using xidata - self.assert_roundtrip_equal([ - # int - sys.maxsize + 1, - -sys.maxsize - 2, - 2**1000, - # tuple - (0, 1.0, []), - (0, 1.0, {}), - (0, 1.0, ([],)), - (0, 1.0, ({},)), - ]) + with ignore_byteswarning(): + for obj in SHAREABLE: + if obj in PICKLEABLE: + self.assert_roundtrip_equal([obj]) + else: + self.assert_not_shareable([obj]) + + def test_not_shareable(self): + with ignore_byteswarning(): + for obj in NOT_SHAREABLE: + if type(obj) is types.MappingProxyType: + self.assert_not_shareable([obj]) + elif obj in PICKLEABLE: + with self.subTest(repr(obj)): + # We don't worry about checking the actual value. + # The other tests should cover that well enough. + got = self.get_roundtrip(obj) + self.assertIs(type(got), type(obj)) + else: + self.assert_not_shareable([obj]) def test_list(self): self.assert_roundtrip_equal_not_identical([ @@ -266,7 +585,7 @@ def assert_class_defs_same(self, defs): if cls not in defs.CLASSES_WITHOUT_EQUALITY: continue instances.append(cls(*args)) - self.assert_roundtrip_not_equal(instances) + self.assert_roundtrip_equal(instances) def assert_class_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. @@ -286,7 +605,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): instances = [] for cls, args in defs.TOP_CLASSES.items(): - with self.subTest(cls): + with self.subTest(repr(cls)): setattr(mod, cls.__name__, cls) xid = self.get_xidata(cls) inst = cls(*args) @@ -295,7 +614,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): (cls, xid, inst, instxid)) for cls, xid, inst, instxid in instances: - with self.subTest(cls): + with self.subTest(repr(cls)): delattr(mod, cls.__name__) if fail: with self.assertRaises(NotShareableError): @@ -403,13 +722,13 @@ def assert_func_defs_same(self, defs): def assert_func_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) self.assert_not_shareable(defs.TOP_FUNCTIONS) def assert_func_defs_other_unpickle(self, defs, mod, *, fail=False): # Unpickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) captured = [] for func in defs.TOP_FUNCTIONS: @@ -434,7 +753,7 @@ def assert_func_defs_not_shareable(self, defs): self.assert_not_shareable(defs.TOP_FUNCTIONS) def test_user_function_normal(self): -# self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) + self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) self.assert_func_defs_same(defs) def test_user_func_in___main__(self): @@ -505,7 +824,7 @@ def test_nested_function(self): # exceptions def test_user_exception_normal(self): - self.assert_roundtrip_not_equal([ + self.assert_roundtrip_equal([ defs.MimimalError('error!'), ]) self.assert_roundtrip_equal_not_identical([ @@ -521,7 +840,7 @@ def test_builtin_exception(self): special = { BaseExceptionGroup: (msg, [caught]), ExceptionGroup: (msg, [caught]), -# UnicodeError: (None, msg, None, None, None), + UnicodeError: (None, msg, None, None, None), UnicodeEncodeError: ('utf-8', '', 1, 3, msg), UnicodeDecodeError: ('utf-8', b'', 1, 3, msg), UnicodeTranslateError: ('', 1, 3, msg), @@ -531,7 +850,7 @@ def test_builtin_exception(self): args = special.get(cls) or (msg,) exceptions.append(cls(*args)) - self.assert_roundtrip_not_equal(exceptions) + self.assert_roundtrip_equal(exceptions) class MarshalTests(_GetXIDataTests): @@ -576,7 +895,7 @@ def test_simple_builtin_objects(self): '', ]) self.assert_not_shareable([ - object(), + OBJECT, types.SimpleNamespace(), ]) @@ -647,10 +966,7 @@ def test_builtin_type(self): shareable = [ StopIteration, ] - types = [ - *BUILTIN_TYPES, - *OTHER_TYPES, - ] + types = BUILTIN_TYPES self.assert_not_shareable(cls for cls in types if cls not in shareable) self.assert_roundtrip_identical(cls for cls in types @@ -763,7 +1079,7 @@ class ShareableFuncTests(_GetXIDataTests): MODE = 'func' def test_stateless(self): - self.assert_roundtrip_not_equal([ + self.assert_roundtrip_equal([ *defs.STATELESS_FUNCTIONS, # Generators can be stateless too. *defs.FUNCTION_LIKE, @@ -912,10 +1228,49 @@ def test_impure_script_function(self): ], expecttype=types.CodeType) +class ShareableFallbackTests(_GetXIDataTests): + + MODE = 'fallback' + + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + + def test_not_shareable(self): + okay = [ + *PICKLEABLE, + *defs.STATELESS_FUNCTIONS, + LAMBDA, + ] + ignored = [ + *TUPLES_WITHOUT_EQUALITY, + OBJECT, + METHOD, + BUILTIN_METHOD, + METHOD_WRAPPER, + ] + with ignore_byteswarning(): + self.assert_roundtrip_equal([ + *(o for o in NOT_SHAREABLE + if o in okay and o not in ignored + and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_roundtrip_not_equal([ + *(o for o in NOT_SHAREABLE + if o in ignored and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_not_shareable([ + *(o for o in NOT_SHAREABLE if o not in okay), + MAPPING_PROXY_EMPTY, + ]) + + class ShareableTypeTests(_GetXIDataTests): MODE = 'xidata' + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + def test_singletons(self): self.assert_roundtrip_identical([ None, @@ -983,8 +1338,8 @@ def test_tuple(self): def test_tuples_containing_non_shareable_types(self): non_shareables = [ - Exception(), - object(), + EXCEPTION, + OBJECT, ] for s in non_shareables: value = tuple([0, 1.0, s]) @@ -999,6 +1354,9 @@ def test_tuples_containing_non_shareable_types(self): # The rest are not shareable. + def test_not_shareable(self): + self.assert_not_shareable(NOT_SHAREABLE) + def test_object(self): self.assert_not_shareable([ object(), @@ -1015,12 +1373,12 @@ def test_function_object(self): for func in defs.FUNCTIONS: assert type(func) is types.FunctionType, func assert type(defs.SpamOkay.okay) is types.FunctionType, func - assert type(lambda: None) is types.LambdaType + assert type(LAMBDA) is types.LambdaType self.assert_not_shareable([ *defs.FUNCTIONS, defs.SpamOkay.okay, - (lambda: None), + LAMBDA, ]) def test_builtin_function(self): @@ -1085,10 +1443,7 @@ def test_class(self): self.assert_not_shareable(instances) def test_builtin_type(self): - self.assert_not_shareable([ - *BUILTIN_TYPES, - *OTHER_TYPES, - ]) + self.assert_not_shareable(BUILTIN_TYPES) def test_exception(self): self.assert_not_shareable([ @@ -1127,7 +1482,7 @@ def test_builtin_objects(self): """, ns, ns) self.assert_not_shareable([ - types.MappingProxyType({}), + MAPPING_PROXY_EMPTY, types.SimpleNamespace(), # types.CellType types.CellType(), diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 172cebcaa4884f..f9fa1dab291056 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -1779,7 +1779,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; @@ -2694,7 +2694,7 @@ add_channelid_type(PyObject *mod) Py_DECREF(cls); return NULL; } - if (ensure_xid_class(cls, _channelid_shared) < 0) { + if (ensure_xid_class(cls, GETDATA(_channelid_shared)) < 0) { Py_DECREF(cls); return NULL; } @@ -2797,12 +2797,12 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (ensure_xid_class(send, _channelend_shared) < 0) { + if (ensure_xid_class(send, GETDATA(_channelend_shared)) < 0) { Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); return -1; } - if (ensure_xid_class(recv, _channelend_shared) < 0) { + if (ensure_xid_class(recv, GETDATA(_channelend_shared)) < 0) { (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 526249a0e1aec3..209fcdfd0cd01e 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1143,7 +1143,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) { _queue_unmark_waiter(queue, queues->mutex); GLOBAL_FREE(data); return -1; @@ -1270,7 +1270,7 @@ set_external_queue_type(module_state *state, PyTypeObject *queue_type) } // Add and register the new type. - if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { + if (ensure_xid_class(queue_type, GETDATA(_queueobj_shared)) < 0) { return -1; } state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index edd65577284a20..d73cbca36359c7 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -5,8 +5,10 @@ _RESOLVE_MODINIT_FUNC_NAME(NAME) +#define GETDATA(FUNC) ((_PyXIData_getdata_t){.basic=FUNC}) + static int -ensure_xid_class(PyTypeObject *cls, xidatafunc getdata) +ensure_xid_class(PyTypeObject *cls, _PyXIData_getdata_t getdata) { PyThreadState *tstate = PyThreadState_Get(); return _PyXIData_RegisterClass(tstate, cls, getdata); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 77678f7c126005..f3c571e717fd0e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -286,7 +286,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) *p_state = cls; // Register XID for the builtin memoryview type. - if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) { + if (ensure_xid_class(&PyMemoryView_Type, GETDATA(_pybuffer_shared)) < 0) { return -1; } // We don't ever bother un-registering memoryview. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 76bd76cc6b2490..136e6a7a015049 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1991,7 +1991,14 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } if (strcmp(mode, "xidata") == 0) { - if (_PyObject_GetXIData(tstate, obj, xidata) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "fallback") == 0) { + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) + { goto error; } } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 725d6009f84014..dc67de4a40849d 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -210,16 +210,16 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ -/* registry of {type -> xidatafunc} */ +/* registry of {type -> _PyXIData_getdata_t} */ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ static void xid_lookup_init(_PyXIData_lookup_t *); static void xid_lookup_fini(_PyXIData_lookup_t *); struct _dlcontext; -static xidatafunc lookup_getdata(struct _dlcontext *, PyObject *); +static _PyXIData_getdata_t lookup_getdata(struct _dlcontext *, PyObject *); #include "crossinterp_data_lookup.h" @@ -343,7 +343,7 @@ _set_xid_lookup_failure(PyThreadState *tstate, PyObject *obj, const char *msg, set_notshareableerror(tstate, cause, 0, msg); } else { - msg = "%S does not support cross-interpreter data"; + msg = "%R does not support cross-interpreter data"; format_notshareableerror(tstate, cause, 0, msg, obj); } } @@ -356,8 +356,8 @@ _PyObject_CheckXIData(PyThreadState *tstate, PyObject *obj) if (get_lookup_context(tstate, &ctx) < 0) { return -1; } - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (!_PyErr_Occurred(tstate)) { _set_xid_lookup_failure(tstate, obj, NULL, NULL); } @@ -388,9 +388,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *xidata) return 0; } -int -_PyObject_GetXIData(PyThreadState *tstate, - PyObject *obj, _PyXIData_t *xidata) +static int +_get_xidata(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, _PyXIData_t *xidata) { PyInterpreterState *interp = tstate->interp; @@ -398,6 +398,7 @@ _PyObject_GetXIData(PyThreadState *tstate, assert(xidata->obj == NULL); if (xidata->data != NULL || xidata->obj != NULL) { _PyErr_SetString(tstate, PyExc_ValueError, "xidata not cleared"); + return -1; } // Call the "getdata" func for the object. @@ -406,8 +407,8 @@ _PyObject_GetXIData(PyThreadState *tstate, return -1; } Py_INCREF(obj); - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (PyErr_Occurred()) { Py_DECREF(obj); return -1; @@ -419,7 +420,9 @@ _PyObject_GetXIData(PyThreadState *tstate, } return -1; } - int res = getdata(tstate, obj, xidata); + int res = getdata.basic != NULL + ? getdata.basic(tstate, obj, xidata) + : getdata.fallback(tstate, obj, fallback, xidata); Py_DECREF(obj); if (res != 0) { PyObject *cause = _PyErr_GetRaisedException(tstate); @@ -439,6 +442,51 @@ _PyObject_GetXIData(PyThreadState *tstate, return 0; } +int +_PyObject_GetXIDataNoFallback(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return _get_xidata(tstate, obj, _PyXIDATA_XIDATA_ONLY, xidata); +} + +int +_PyObject_GetXIData(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) +{ + switch (fallback) { + case _PyXIDATA_XIDATA_ONLY: + return _get_xidata(tstate, obj, fallback, xidata); + case _PyXIDATA_FULL_FALLBACK: + if (_get_xidata(tstate, obj, fallback, xidata) == 0) { + return 0; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (PyFunction_Check(obj)) { + if (_PyFunction_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + _PyErr_Clear(tstate); + } + // We could try _PyMarshal_GetXIData() but we won't for now. + if (_PyPickle_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + // Raise the original exception. + _PyErr_SetRaisedException(tstate, exc); + return -1; + default: +#ifdef Py_DEBUG + Py_FatalError("unsupported xidata fallback option"); +#endif + _PyErr_SetString(tstate, PyExc_SystemError, + "unsupported xidata fallback option"); + return -1; + } +} + /* pickle C-API */ @@ -1617,14 +1665,9 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyThreadState *tstate = _PyThreadState_GET(); assert(!PyErr_Occurred()); + assert(code != _PyXI_ERR_NO_ERROR); + assert(code != _PyXI_ERR_UNCAUGHT_EXCEPTION); switch (code) { - case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; - case _PyXI_ERR_UNCAUGHT_EXCEPTION: - // There is nothing to apply. -#ifdef Py_DEBUG - Py_UNREACHABLE(); -#endif - return 0; case _PyXI_ERR_OTHER: // XXX msg? PyErr_SetNone(PyExc_InterpreterError); @@ -1649,7 +1692,7 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) break; default: #ifdef Py_DEBUG - Py_UNREACHABLE(); + Py_FatalError("unsupported error code"); #else PyErr_Format(PyExc_RuntimeError, "unsupported error code %d", code); #endif @@ -1796,7 +1839,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); - if (_PyObject_GetXIData(tstate, value, item->xidata) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; // The caller may want to propagate PyExc_NotShareableError diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index d69927dbcd387f..88eb41da89ee40 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -12,7 +12,8 @@ typedef _PyXIData_regitem_t dlregitem_t; // forward static void _xidregistry_init(dlregistry_t *); static void _xidregistry_fini(dlregistry_t *); -static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *); +static _PyXIData_getdata_t _lookup_getdata_from_registry( + dlcontext_t *, PyObject *); /* used in crossinterp.c */ @@ -49,7 +50,7 @@ get_lookup_context(PyThreadState *tstate, dlcontext_t *res) return 0; } -static xidatafunc +static _PyXIData_getdata_t lookup_getdata(dlcontext_t *ctx, PyObject *obj) { /* Cross-interpreter objects are looked up by exact match on the class. @@ -88,24 +89,24 @@ _PyXIData_FormatNotShareableError(PyThreadState *tstate, } -xidatafunc +_PyXIData_getdata_t _PyXIData_Lookup(PyThreadState *tstate, PyObject *obj) { dlcontext_t ctx; if (get_lookup_context(tstate, &ctx) < 0) { - return NULL; + return (_PyXIData_getdata_t){0}; } return lookup_getdata(&ctx, obj); } /***********************************************/ -/* a registry of {type -> xidatafunc} */ +/* a registry of {type -> _PyXIData_getdata_t} */ /***********************************************/ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ /* registry lifecycle */ @@ -200,7 +201,7 @@ _xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls) return NULL; } -static xidatafunc +static _PyXIData_getdata_t _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) { PyTypeObject *cls = Py_TYPE(obj); @@ -209,10 +210,12 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) _xidregistry_lock(xidregistry); dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); - xidatafunc func = matched != NULL ? matched->getdata : NULL; + _PyXIData_getdata_t getdata = matched != NULL + ? matched->getdata + : (_PyXIData_getdata_t){0}; _xidregistry_unlock(xidregistry); - return func; + return getdata; } @@ -220,12 +223,13 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) static int _xidregistry_add_type(dlregistry_t *xidregistry, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t)); if (newhead == NULL) { return -1; } + assert((getdata.basic == NULL) != (getdata.fallback == NULL)); *newhead = (dlregitem_t){ // We do not keep a reference, to avoid keeping the class alive. .cls = cls, @@ -283,13 +287,13 @@ _xidregistry_clear(dlregistry_t *xidregistry) int _PyXIData_RegisterClass(PyThreadState *tstate, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); return -1; } - if (getdata == NULL) { + if (getdata.basic == NULL && getdata.fallback == NULL) { PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); return -1; } @@ -304,7 +308,8 @@ _PyXIData_RegisterClass(PyThreadState *tstate, dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { - assert(matched->getdata == getdata); + assert(matched->getdata.basic == getdata.basic); + assert(matched->getdata.fallback == getdata.fallback); matched->refcount += 1; goto finally; } @@ -608,7 +613,8 @@ _tuple_shared_free(void* data) } static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) +_tuple_shared(PyThreadState *tstate, PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) { Py_ssize_t len = PyTuple_GET_SIZE(obj); if (len < 0) { @@ -636,7 +642,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) int res = -1; if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetXIData(tstate, item, xidata_i); + res = _PyObject_GetXIData(tstate, item, fallback, xidata_i); _Py_LeaveRecursiveCallTstate(tstate); } if (res < 0) { @@ -737,40 +743,48 @@ _PyFunction_GetXIData(PyThreadState *tstate, PyObject *func, static void _register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry) { +#define REGISTER(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.basic=(GETDATA)})) +#define REGISTER_FALLBACK(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.fallback=(GETDATA)})) // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (REGISTER(Py_TYPE(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (REGISTER(&PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _PyBytes_GetXIData) != 0) { + if (REGISTER(&PyBytes_Type, _PyBytes_GetXIData) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (REGISTER(&PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } // bool - if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { + if (REGISTER(&PyBool_Type, _bool_shared) != 0) { Py_FatalError("could not register bool for cross-interpreter sharing"); } // float - if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { + if (REGISTER(&PyFloat_Type, _float_shared) != 0) { Py_FatalError("could not register float for cross-interpreter sharing"); } // tuple - if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + if (REGISTER_FALLBACK(&PyTuple_Type, _tuple_shared) != 0) { Py_FatalError("could not register tuple for cross-interpreter sharing"); } // For now, we do not register PyCode_Type or PyFunction_Type. +#undef REGISTER +#undef REGISTER_FALLBACK } From c612a4d2c067aedc6b8bc3fc694758cd16ec4944 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 17:13:13 +0200 Subject: [PATCH 181/713] [3.14] gh-133982: Update test_bufio to use self.open (gh-133983) (gh-134428) (cherry picked from commit 84d5f8d799dbbf86248375b6edbbcf4a022788c4) Co-authored-by: Cody Maloney --- Lib/test/test_bufio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_bufio.py b/Lib/test/test_bufio.py index dc9a82dc635318..cb9cb4d0bc7e9c 100644 --- a/Lib/test/test_bufio.py +++ b/Lib/test/test_bufio.py @@ -28,7 +28,7 @@ def try_one(self, s): f.write(b"\n") f.write(s) f.close() - f = open(os_helper.TESTFN, "rb") + f = self.open(os_helper.TESTFN, "rb") line = f.readline() self.assertEqual(line, s + b"\n") line = f.readline() From 74dde92903769b77843152a3603f0e50845ccc81 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 17:13:43 +0200 Subject: [PATCH 182/713] [3.14] gh-71253: Match _io exception in _pyio (gh-133985) (gh-134430) Test was only testing _io, expanded to cover _pyio. (cherry picked from commit 06eaf4055c1d7359e129efb65b94f34d2ec51a57) Co-authored-by: Cody Maloney Co-authored-by: Peter Bierma --- Lib/_pyio.py | 3 ++- Lib/test/test_io.py | 4 ++-- .../Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst | 3 +++ 3 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a870de5b532542..f79674fb7a9f5e 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -1563,7 +1563,8 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: - raise OSError('Negative file descriptor') + # bpo-27066: Raise a ValueError for bad value. + raise ValueError(f'opener returned {fd}') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 90680c6d47ab41..4625e3a01faa7b 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -918,7 +918,7 @@ def test_bad_opener_negative_1(self): def badopener(fname, flags): return -1 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -1') def test_bad_opener_other_negative(self): @@ -926,7 +926,7 @@ def test_bad_opener_other_negative(self): def badopener(fname, flags): return -2 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') def test_opener_invalid_fd(self): diff --git a/Misc/NEWS.d/next/Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst b/Misc/NEWS.d/next/Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst new file mode 100644 index 00000000000000..714d707f488709 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst @@ -0,0 +1,3 @@ +Raise :exc:`ValueError` in :func:`open` if *opener* returns a negative +file-descriptor in the Python implementation of :mod:`io` to match the +C implementation. From fade04e0f441be0ea3ec0ef99dd0af8113df9ed8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 17:22:44 +0200 Subject: [PATCH 183/713] [3.14] gh-133982: Run unclosed file test on all io implementations (gh-134165) (gh-134433) Update `test_io` `_check_warn_on_dealloc` to use `self.` to dispatch to different I/O implementations. Update the `_pyio` implementation to match expected behavior, using the same `_dealloc_warn` design as the C implementation uses to report the topmost `__del__` object. The FileIO one now matches all the others, so can use IOBase. There was a missing check on closing (self._fd must be valid), add that check (cherry picked from commit 5b0e82752120a5dc66ce6ee778751d71ba2c33b2) Co-authored-by: Cody Maloney --- Lib/_pyio.py | 18 ++++++++++++++---- Lib/test/test_io.py | 4 ++-- ...5-05-17-20-23-57.gh-issue-133982.smS7au.rst | 3 +++ 3 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-17-20-23-57.gh-issue-133982.smS7au.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index f79674fb7a9f5e..300d5b3cbce32b 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -407,6 +407,9 @@ def __del__(self): if closed: return + if dealloc_warn := getattr(self, "_dealloc_warn", None): + dealloc_warn(self) + # If close() fails, the caller logs the exception with # sys.unraisablehook. close() must be called at the end at __del__(). self.close() @@ -853,6 +856,10 @@ def __repr__(self): else: return "<{}.{} name={!r}>".format(modname, clsname, name) + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.raw, "_dealloc_warn", None): + dealloc_warn(source) + ### Lower-level APIs ### def fileno(self): @@ -1601,12 +1608,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): raise self._fd = fd - def __del__(self): + def _dealloc_warn(self, source): if self._fd >= 0 and self._closefd and not self.closed: import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, + warnings.warn(f'unclosed file {source!r}', ResourceWarning, stacklevel=2, source=self) - self.close() def __getstate__(self): raise TypeError(f"cannot pickle {self.__class__.__name__!r} object") @@ -1781,7 +1787,7 @@ def close(self): if not self.closed: self._stat_atopen = None try: - if self._closefd: + if self._closefd and self._fd >= 0: os.close(self._fd) finally: super().close() @@ -2690,6 +2696,10 @@ def readline(self, size=None): def newlines(self): return self._decoder.newlines if self._decoder else None + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None): + dealloc_warn(source) + class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 4625e3a01faa7b..aa619a96ab7535 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -4417,7 +4417,7 @@ def test_abc_inheritance_official(self): self._check_abc_inheritance(io) def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) + f = self.open(*args, **kwargs) r = repr(f) with self.assertWarns(ResourceWarning) as cm: f = None @@ -4446,7 +4446,7 @@ def cleanup_fds(): r, w = os.pipe() fds += r, w with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) + self.open(r, *args, closefd=False, **kwargs) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): diff --git a/Misc/NEWS.d/next/Library/2025-05-17-20-23-57.gh-issue-133982.smS7au.rst b/Misc/NEWS.d/next/Library/2025-05-17-20-23-57.gh-issue-133982.smS7au.rst new file mode 100644 index 00000000000000..a6753145981181 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-17-20-23-57.gh-issue-133982.smS7au.rst @@ -0,0 +1,3 @@ +Emit :exc:`RuntimeWarning` in the Python implementation of :mod:`io` when +the :term:`file-like object ` is not closed explicitly in the +presence of multiple I/O layers. From 17bf6ab0c1b4189dd5a201d952a00609800fbc19 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 17:57:58 +0200 Subject: [PATCH 184/713] [3.14] gh-62184: Remove _pyio import of _io.FileIO (gh-134192) (gh-134437) This was added in the add of `_io`, isn't used since bpo-21859 when a `_pyio` implementation was added which defines `FileIO` lower down in the file. (cherry picked from commit 0a68068bd2a5bff98998067a141b17af5be9b750) Co-authored-by: Cody Maloney --- Lib/_pyio.py | 2 -- .../next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 300d5b3cbce32b..fb2a6d049caab6 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -648,8 +648,6 @@ def write(self, b): self._unsupported("write") io.RawIOBase.register(RawIOBase) -from _io import FileIO -RawIOBase.register(FileIO) class BufferedIOBase(IOBase): diff --git a/Misc/NEWS.d/next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst b/Misc/NEWS.d/next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst new file mode 100644 index 00000000000000..7bc994e57fb58f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst @@ -0,0 +1,2 @@ +Remove import of C implementation of :class:`io.FileIO` from Python +implementation which has its own implementation From cdc92cd9fc17f6e06c51584cde2ab6cf21f188d4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 18:19:25 +0200 Subject: [PATCH 185/713] [3.14] gh-132983: Refactor shared code in train_dict and finalize_dict (GH-134432) (#134442) gh-132983: Refactor shared code in train_dict and finalize_dict (GH-134432) Refactor shared code in train_dict and finalize_dict (cherry picked from commit c64a21454b3c139af9e88941a286885fc4828a7e) Co-authored-by: Emma Smith --- Modules/_zstd/_zstdmodule.c | 123 ++++++++++++++++-------------------- 1 file changed, 55 insertions(+), 68 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 0294828aa106ea..b2e4f95b906356 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -172,6 +172,49 @@ get_zstd_state(PyObject *module) return (_zstd_state *)state; } +static Py_ssize_t +calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, + size_t **chunk_sizes) +{ + Py_ssize_t chunks_number; + Py_ssize_t sizes_sum; + Py_ssize_t i; + + chunks_number = Py_SIZE(samples_sizes); + if ((size_t) chunks_number > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, + "The number of samples should be <= %u.", UINT32_MAX); + return -1; + } + + /* Prepare chunk_sizes */ + *chunk_sizes = PyMem_New(size_t, chunks_number); + if (*chunk_sizes == NULL) { + PyErr_NoMemory(); + return -1; + } + + sizes_sum = 0; + for (i = 0; i < chunks_number; i++) { + PyObject *size = PyTuple_GetItem(samples_sizes, i); + (*chunk_sizes)[i] = PyLong_AsSize_t(size); + if ((*chunk_sizes)[i] == (size_t)-1 && PyErr_Occurred()) { + PyErr_Format(PyExc_ValueError, + "Items in samples_sizes should be an int " + "object, with a value between 0 and %u.", SIZE_MAX); + return -1; + } + sizes_sum += (*chunk_sizes)[i]; + } + + if (sizes_sum != Py_SIZE(samples_bytes)) { + PyErr_SetString(PyExc_ValueError, + "The samples size tuple doesn't match the concatenation's size."); + return -1; + } + return chunks_number; +} + /*[clinic input] _zstd.train_dict @@ -192,14 +235,10 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size) /*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/ { - // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict - // are pretty similar. We should see if we can refactor them to share that code. - Py_ssize_t chunks_number; - size_t *chunk_sizes = NULL; PyObject *dst_dict_bytes = NULL; + size_t *chunk_sizes = NULL; + Py_ssize_t chunks_number; size_t zstd_ret; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { @@ -207,39 +246,14 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, return NULL; } - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) + { return NULL; } - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { - PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); - goto error; - } - /* Allocate dict buffer */ dst_dict_bytes = PyBytes_FromStringAndSize(NULL, dict_size); if (dst_dict_bytes == NULL) { @@ -307,8 +321,6 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, PyObject *dst_dict_bytes = NULL; size_t zstd_ret; ZDICT_params_t params; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { @@ -316,39 +328,14 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, return NULL; } - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) + { return NULL; } - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { - PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); - goto error; - } - /* Allocate dict buffer */ dst_dict_bytes = PyBytes_FromStringAndSize(NULL, dict_size); if (dst_dict_bytes == NULL) { From 6ea83e077e0a06261d9e72634ed76d6fafe72297 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 18:43:07 +0200 Subject: [PATCH 186/713] [3.14] gh-134026: Fix grammar description of for statement (GH-134034) (GH-134424) gh-134026: Fix grammar description of for statement (GH-134034) (cherry picked from commit 4eacf3883dd041c31133ea407204b797a17559b1) Co-authored-by: Yash Vijay --- Doc/reference/compound_stmts.rst | 14 +++++++------- Doc/reference/expressions.rst | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 5d4298f70e0e14..e95fa3a6424e23 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -154,15 +154,15 @@ The :keyword:`for` statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: .. productionlist:: python-grammar - for_stmt: "for" `target_list` "in" `starred_list` ":" `suite` + for_stmt: "for" `target_list` "in" `starred_expression_list` ":" `suite` : ["else" ":" `suite`] -The ``starred_list`` expression is evaluated once; it should yield an -:term:`iterable` object. An :term:`iterator` is created for that iterable. -The first item provided -by the iterator is then assigned to the target list using the standard -rules for assignments (see :ref:`assignment`), and the suite is executed. This -repeats for each item provided by the iterator. When the iterator is exhausted, +The :token:`~python-grammar:starred_expression_list` expression is evaluated +once; it should yield an :term:`iterable` object. An :term:`iterator` is +created for that iterable. The first item provided by the iterator is then +assigned to the target list using the standard rules for assignments +(see :ref:`assignment`), and the suite is executed. This repeats for each +item provided by the iterator. When the iterator is exhausted, the suite in the :keyword:`!else` clause, if present, is executed, and the loop terminates. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8837344e5ddca1..2a550b504ca765 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -1928,7 +1928,7 @@ Expression lists single: , (comma); expression list .. productionlist:: python-grammar - starred_expression: ["*"] `or_expr` + starred_expression: "*" `or_expr` | `expression` flexible_expression: `assignment_expression` | `starred_expression` flexible_expression_list: `flexible_expression` ("," `flexible_expression`)* [","] starred_expression_list: `starred_expression` ("," `starred_expression`)* [","] From df6d9e7fbdbb583c12a627f202f810f4772e7930 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 18:50:02 +0200 Subject: [PATCH 187/713] [3.14] gh-80050: Update BufferedReader.read docs around non-blocking (GH-130653) (#134444) gh-80050: Update BufferedReader.read docs around non-blocking (GH-130653) (cherry picked from commit e1f891414b2329414a6160ed246f5f869a218bfd) Co-authored-by: Cody Maloney --- Doc/library/io.rst | 85 ++++++++++++++++++++++------------------------ 1 file changed, 41 insertions(+), 44 deletions(-) diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 08c76da3d8c00a..de5cab5aee649f 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -528,14 +528,13 @@ I/O Base Classes It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, - :meth:`readinto` and :meth:`write` will try (respectively) to read as much - input as requested or to consume all given output, at the expense of - making perhaps more than one system call. + :meth:`readinto` and :meth:`write` will try (respectively) to read + as much input as requested or to emit all provided data. - In addition, those methods can raise :exc:`BlockingIOError` if the - underlying raw stream is in non-blocking mode and cannot take or give - enough data; unlike their :class:`RawIOBase` counterparts, they will - never return ``None``. + In addition, if the underlying raw stream is in non-blocking mode, when the + system returns would block :meth:`write` will raise :exc:`BlockingIOError` + with :attr:`BlockingIOError.characters_written` and :meth:`read` will return + data read so far or ``None`` if no data is available. Besides, the :meth:`read` method does not have a default implementation that defers to :meth:`readinto`. @@ -568,29 +567,40 @@ I/O Base Classes .. method:: read(size=-1, /) - Read and return up to *size* bytes. If the argument is omitted, ``None``, - or negative, data is read and returned until EOF is reached. An empty - :class:`bytes` object is returned if the stream is already at EOF. + Read and return up to *size* bytes. If the argument is omitted, ``None``, + or negative read as much as possible. - If the argument is positive, and the underlying raw stream is not - interactive, multiple raw reads may be issued to satisfy the byte count - (unless EOF is reached first). But for interactive raw streams, at most - one raw read will be issued, and a short result does not imply that EOF is - imminent. + Fewer bytes may be returned than requested. An empty :class:`bytes` object + is returned if the stream is already at EOF. More than one read may be + made and calls may be retried if specific errors are encountered, see + :meth:`os.read` and :pep:`475` for more details. Less than size bytes + being returned does not imply that EOF is imminent. - A :exc:`BlockingIOError` is raised if the underlying raw stream is in - non blocking-mode, and has no data available at the moment. + When reading as much as possible the default implementation will use + ``raw.readall`` if available (which should implement + :meth:`RawIOBase.readall`), otherwise will read in a loop until read + returns ``None``, an empty :class:`bytes`, or a non-retryable error. For + most streams this is to EOF, but for non-blocking streams more data may + become available. + + .. note:: + + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`io` implementations return ``None``. .. method:: read1(size=-1, /) - Read and return up to *size* bytes, with at most one call to the - underlying raw stream's :meth:`~RawIOBase.read` (or - :meth:`~RawIOBase.readinto`) method. This can be useful if you are - implementing your own buffering on top of a :class:`BufferedIOBase` - object. + Read and return up to *size* bytes, calling :meth:`~RawIOBase.readinto` + which may retry if :py:const:`~errno.EINTR` is encountered per + :pep:`475`. If *size* is ``-1`` or not provided, the implementation will + choose an arbitrary value for *size*. - If *size* is ``-1`` (the default), an arbitrary number of bytes are - returned (more than zero unless EOF is reached). + .. note:: + + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`io` implementations return ``None``. .. method:: readinto(b, /) @@ -767,34 +777,21 @@ than raw I/O does. .. method:: peek(size=0, /) - Return bytes from the stream without advancing the position. At most one - single read on the raw stream is done to satisfy the call. The number of - bytes returned may be less or more than requested. + Return bytes from the stream without advancing the position. The number of + bytes returned may be less or more than requested. If the underlying raw + stream is non-blocking and the operation would block, returns empty bytes. .. method:: read(size=-1, /) - Read and return *size* bytes, or if *size* is not given or negative, until - EOF or if the read call would block in non-blocking mode. - - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read` .. method:: read1(size=-1, /) - Read and return up to *size* bytes with only one call on the raw stream. - If at least one byte is buffered, only buffered bytes are returned. - Otherwise, one raw stream read call is made. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read1` .. versionchanged:: 3.7 The *size* argument is now optional. - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. - .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non @@ -826,8 +823,8 @@ than raw I/O does. Write the :term:`bytes-like object`, *b*, and return the number of bytes written. When in non-blocking mode, a - :exc:`BlockingIOError` is raised if the buffer needs to be written out but - the raw stream blocks. + :exc:`BlockingIOError` with :attr:`BlockingIOError.characters_written` set + is raised if the buffer needs to be written out but the raw stream blocks. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) From f8ced87fa3d2d91be9a445dbfac9def3df527cb3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 19:36:32 +0200 Subject: [PATCH 188/713] [3.14] gh-132124: improve safety nets for creating AF_UNIX socket files (GH-134085) (#134447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-132124: improve safety nets for creating AF_UNIX socket files (GH-134085) * ensure that we can create AF_UNIX socket files * emit a warning if system-wide temporary directory is used (cherry picked from commit 1a07a01014bde23acd2684916ef38dc0cd73c2de) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/multiprocessing/connection.py | 2 +- Lib/multiprocessing/util.py | 79 ++++++++++++++++++- Lib/tempfile.py | 5 +- ...-05-16-12-40-37.gh-issue-132124.T_5Odx.rst | 6 ++ 4 files changed, 87 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-16-12-40-37.gh-issue-132124.T_5Odx.rst diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 5f288a8d393240..fc00d2861260a8 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -76,7 +76,7 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + return tempfile.mktemp(prefix='sock-', dir=util.get_temp_dir()) elif family == 'AF_PIPE': return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % (os.getpid(), next(_mmap_counter)), dir="") diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b7192042b9cf47..a1a537dd48dea7 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -19,7 +19,7 @@ from . import process __all__ = [ - 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', @@ -34,6 +34,7 @@ DEBUG = 10 INFO = 20 SUBWARNING = 25 +WARNING = 30 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' @@ -53,6 +54,10 @@ def info(msg, *args): if _logger: _logger.log(INFO, msg, *args, stacklevel=2) +def warn(msg, *args): + if _logger: + _logger.log(WARNING, msg, *args, stacklevel=2) + def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args, stacklevel=2) @@ -121,6 +126,21 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # +# Maximum length of a socket file path is usually between 92 and 108 [1], +# but Linux is known to use a size of 108 [2]. BSD-based systems usually +# use a size of 104 or 108 and Windows does not create AF_UNIX sockets. +# +# [1]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [2]: https://man7.org/linux/man-pages/man7/unix.7.html. + +if sys.platform == 'linux': + _SUN_PATH_MAX = 108 +elif sys.platform.startswith(('openbsd', 'freebsd')): + _SUN_PATH_MAX = 104 +else: + # On Windows platforms, we do not create AF_UNIX sockets. + _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -130,12 +150,67 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None +def _get_base_temp_dir(tempfile): + """Get a temporary directory where socket files will be created. + + To prevent additional imports, pass a pre-imported 'tempfile' module. + """ + if os.name == 'nt': + return None + # Most of the time, the default temporary directory is /tmp. Thus, + # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do + # not have a path length exceeding SUN_PATH_MAX. + # + # If users specify their own temporary directory, we may be unable + # to create those files. Therefore, we fall back to the system-wide + # temporary directory /tmp, assumed to exist on POSIX systems. + # + # See https://github.com/python/cpython/issues/132124. + base_tempdir = tempfile.gettempdir() + # Files created in a temporary directory are suffixed by a string + # generated by tempfile._RandomNameSequence, which, by design, + # is 8 characters long. + # + # Thus, the length of socket filename will be: + # + # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') + sun_path_len = len(base_tempdir) + 14 + 14 + if sun_path_len <= _SUN_PATH_MAX: + return base_tempdir + # Fallback to the default system-wide temporary directory. + # This ignores user-defined environment variables. + # + # On POSIX systems, /tmp MUST be writable by any application [1]. + # We however emit a warning if this is not the case to prevent + # obscure errors later in the execution. + # + # On some legacy systems, /var/tmp and /usr/tmp can be present + # and will be used instead. + # + # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html + dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] + try: + base_system_tempdir = tempfile._get_default_tempdir(dirlist) + except FileNotFoundError: + warn("Process-wide temporary directory %s will not be usable for " + "creating socket files and no usable system-wide temporary " + "directory was found in %s", base_tempdir, dirlist) + # At this point, the system-wide temporary directory is not usable + # but we may assume that the user-defined one is, even if we will + # not be able to write socket files out there. + return base_tempdir + warn("Ignoring user-defined temporary directory: %s", base_tempdir) + # at most max(map(len, dirlist)) + 14 + 14 = 36 characters + assert len(base_system_tempdir) + 14 + 14 <= _SUN_PATH_MAX + return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile - tempdir = tempfile.mkdtemp(prefix='pymp-') + base_tempdir = _get_base_temp_dir(tempfile) + tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer # can be called late during Python shutdown diff --git a/Lib/tempfile.py b/Lib/tempfile.py index cadb0bed3cce3b..5e3ccab5f48502 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -180,7 +180,7 @@ def _candidate_tempdir_list(): return dirlist -def _get_default_tempdir(): +def _get_default_tempdir(dirlist=None): """Calculate the default directory to use for temporary files. This routine should be called exactly once. @@ -190,7 +190,8 @@ def _get_default_tempdir(): service, the name of the test file must be randomized.""" namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() + if dirlist is None: + dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: diff --git a/Misc/NEWS.d/next/Library/2025-05-16-12-40-37.gh-issue-132124.T_5Odx.rst b/Misc/NEWS.d/next/Library/2025-05-16-12-40-37.gh-issue-132124.T_5Odx.rst new file mode 100644 index 00000000000000..acf3577ece4e9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-16-12-40-37.gh-issue-132124.T_5Odx.rst @@ -0,0 +1,6 @@ +On POSIX-compliant systems, :func:`!multiprocessing.util.get_temp_dir` now +ignores :envvar:`TMPDIR` (and similar environment variables) if the path +length of ``AF_UNIX`` socket files exceeds the platform-specific maximum +length when using the :ref:`forkserver +` start method. Patch by Bénédikt +Tran. From 11d77d21120b77ddfd97f471c0d7a31745141131 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 21:35:31 +0200 Subject: [PATCH 189/713] [3.14] gh-132983: Fix refleak in zstd dictionary functions (gh-134463) (cherry picked from commit fb68776591485, AKA gh-134459) Co-authored-by: Emma Smith --- Modules/_zstd/_zstdmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index b2e4f95b906356..17d3bff1e98769 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -251,7 +251,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, &chunk_sizes); if (chunks_number < 0) { - return NULL; + goto error; } /* Allocate dict buffer */ @@ -333,7 +333,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, &chunk_sizes); if (chunks_number < 0) { - return NULL; + goto error; } /* Allocate dict buffer */ From 93ab55bd703f1238cc93327a1a285921b47fa8a3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 21 May 2025 22:43:29 +0200 Subject: [PATCH 190/713] [3.14] gh-132775: Use _PyFunction_VerifyStateless() and _PyCode_VerifyStateless() (gh-134465) (cherry picked from commit a66bae8bb, AKA gh-134439) Co-authored-by: Eric Snow --- Include/internal/pycore_pyerrors.h | 4 +- Lib/test/test__interpreters.py | 3 +- Modules/_interpretersmodule.c | 139 ++++++++++++++--------------- 3 files changed, 70 insertions(+), 76 deletions(-) diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f357b88e220e6e..2c2048f7e1272a 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -94,13 +94,13 @@ extern void _PyErr_Fetch( PyObject **value, PyObject **traceback); -extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetRaisedException(PyThreadState *tstate); PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); -extern void _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); +PyAPI_FUNC(void) _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); extern void _PyErr_Restore( PyThreadState *tstate, diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 0c43f46300f67d..63fdaad8de7ef5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -1054,7 +1054,7 @@ def test_closure(self): def script(): assert spam - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): _interpreters.run_func(self.id, script) # XXX This hasn't been fixed yet. @@ -1065,6 +1065,7 @@ def script(): with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) + @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index f3c571e717fd0e..91cd92806206be 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -8,6 +8,8 @@ #include "Python.h" #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_function.h" // _PyFunction_VerifyStateless() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -374,34 +376,17 @@ check_code_str(PyUnicodeObject *text) return NULL; } -static const char * -check_code_object(PyCodeObject *code) +#ifndef NDEBUG +static int +code_has_args(PyCodeObject *code) { assert(code != NULL); - if (code->co_argcount > 0 + return (code->co_argcount > 0 || code->co_posonlyargcount > 0 || code->co_kwonlyargcount > 0 - || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) - { - return "arguments not supported"; - } - if (code->co_ncellvars > 0) { - return "closures not supported"; - } - // We trust that no code objects under co_consts have unbound cell vars. - - if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { - return "only basic functions are supported"; - } - if (code->_co_monitoring != NULL) { - return "only basic functions are supported"; - } - if (code->co_extra != NULL) { - return "only basic functions are supported"; - } - - return NULL; + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)); } +#endif #define RUN_TEXT 1 #define RUN_CODE 2 @@ -429,8 +414,10 @@ get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) flags = RUN_TEXT; } else { - assert(PyCode_Check(arg) - && (check_code_object((PyCodeObject *)arg) == NULL)); + assert(PyCode_Check(arg)); + assert(_PyCode_VerifyStateless( + PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0); + assert(!code_has_args((PyCodeObject *)arg)); flags = RUN_CODE; // Serialize the code object. @@ -949,7 +936,8 @@ Bind the given attributes in the interpreter's __main__ module."); static PyUnicodeObject * -convert_script_arg(PyObject *arg, const char *fname, const char *displayname, +convert_script_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, const char *expected) { PyUnicodeObject *str = NULL; @@ -968,8 +956,8 @@ convert_script_arg(PyObject *arg, const char *fname, const char *displayname, const char *err = check_code_str(str); if (err != NULL) { Py_DECREF(str); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad script text (%s)", fname, err); + _PyErr_Format(tstate, PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); return NULL; } @@ -977,51 +965,44 @@ convert_script_arg(PyObject *arg, const char *fname, const char *displayname, } static PyCodeObject * -convert_code_arg(PyObject *arg, const char *fname, const char *displayname, +convert_code_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, const char *expected) { - const char *kind = NULL; + PyObject *cause; PyCodeObject *code = NULL; if (PyFunction_Check(arg)) { - if (PyFunction_GetClosure(arg) != NULL) { - PyErr_Format(PyExc_ValueError, - "%.200s(): closures not supported", fname); - return NULL; - } - code = (PyCodeObject *)PyFunction_GetCode(arg); - if (code == NULL) { - if (PyErr_Occurred()) { - // This chains. - PyErr_Format(PyExc_ValueError, - "%.200s(): bad func", fname); - } - else { - PyErr_Format(PyExc_ValueError, - "%.200s(): func.__code__ missing", fname); - } - return NULL; + // For now we allow globals, so we can't use + // _PyFunction_VerifyStateless(). + PyObject *codeobj = PyFunction_GetCode(arg); + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) { + goto chained; } - Py_INCREF(code); - kind = "func"; + code = (PyCodeObject *)Py_NewRef(codeobj); } else if (PyCode_Check(arg)) { + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) { + goto chained; + } code = (PyCodeObject *)Py_NewRef(arg); - kind = "code object"; } else { _PyArg_BadArgument(fname, displayname, expected, arg); return NULL; } - const char *err = check_code_object(code); - if (err != NULL) { - Py_DECREF(code); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad %s (%s)", fname, kind, err); - return NULL; - } - return code; + +chained: + cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + _PyArg_BadArgument(fname, displayname, expected, arg); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); + _PyErr_SetRaisedException(tstate, exc); + return NULL; } static int @@ -1057,12 +1038,14 @@ _interp_exec(PyObject *self, PyInterpreterState *interp, static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".exec" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; PyObject *id, *code; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, + "OO|O$p:" FUNCNAME, kwlist, &id, &code, &shared, &restricted)) { return NULL; @@ -1077,12 +1060,12 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) const char *expected = "a string, a function, or a code object"; if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); + code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME, + "argument 2", expected); } else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); + code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME, + "argument 2", expected); } if (code == NULL) { return NULL; @@ -1096,6 +1079,7 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return excinfo; } Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(exec_doc, @@ -1118,13 +1102,15 @@ is ignored, including its __globals__ dict."); static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".run_string" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; PyObject *id, *script; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O$p:" MODULE_NAME_STR ".run_string", - kwlist, &id, &script, &shared, &restricted)) + "OU|O$p:" FUNCNAME, kwlist, + &id, &script, &shared, &restricted)) { return NULL; } @@ -1136,7 +1122,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".run_string", + script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME, "argument 2", "a string"); if (script == NULL) { return NULL; @@ -1150,6 +1136,7 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return excinfo; } Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(run_string_doc, @@ -1162,13 +1149,15 @@ Execute the provided string in the identified interpreter.\n\ static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".run_func" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; PyObject *id, *func; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".run_func", - kwlist, &id, &func, &shared, &restricted)) + "OO|O$p:" FUNCNAME, kwlist, + &id, &func, &shared, &restricted)) { return NULL; } @@ -1180,7 +1169,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", + PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME, "argument 2", "a function or a code object"); if (code == NULL) { @@ -1195,6 +1184,7 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return excinfo; } Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(run_func_doc, @@ -1209,6 +1199,8 @@ are not supported. Methods and other callables are not supported either.\n\ static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".call" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "callable", "args", "kwargs", "restrict", NULL}; PyObject *id, *callable; @@ -1216,7 +1208,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) PyObject *kwargs_obj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, + "OO|OO$p:" FUNCNAME, kwlist, &id, &callable, &args_obj, &kwargs_obj, &restricted)) { @@ -1231,15 +1223,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) } if (args_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected args"); + _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected args"); return NULL; } if (kwargs_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + _PyErr_SetString(tstate, PyExc_ValueError, "got unexpected kwargs"); return NULL; } - PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", + PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME, "argument 2", "a function"); if (code == NULL) { return NULL; @@ -1253,6 +1245,7 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return excinfo; } Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(call_doc, From 998991513b3300c3c6627857df92db81a40355c9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 03:33:35 +0200 Subject: [PATCH 191/713] [3.14] gh-127960 Fix the REPL to set the correct namespace by setting the correct `__main__` module (gh-134275) (gh-134473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `__main__` module imported in the `_pyrepl` module points to the `_pyrepl` module itself when the interpreter was launched without `-m` option and didn't execute a module, while it's an unexpected behavior that `__main__` can be `_pyrepl` and relative imports such as `from . import *` works based on the `_pyrepl` module. (cherry picked from commit b1b8962443e7d418601658a4b05347a5a9161910) Co-authored-by: Yuichiro Tachibana (Tsuchiya) Co-authored-by: Łukasz Langa --- Lib/_pyrepl/_module_completer.py | 4 +- Lib/_pyrepl/main.py | 11 ++- Lib/_pyrepl/readline.py | 1 + Lib/test/support/__init__.py | 6 -- Lib/test/test_pyrepl/test_pyrepl.py | 78 +++++++++++++++---- ...-05-21-18-02-56.gh-issue-127960.W3J_2X.rst | 3 + Modules/main.c | 16 +++- 7 files changed, 85 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 9aafb55090e2ce..494a501101a9b2 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -17,8 +17,8 @@ def make_default_module_completer() -> ModuleCompleter: - # Inside pyrepl, __package__ is set to '_pyrepl' - return ModuleCompleter(namespace={'__package__': '_pyrepl'}) + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) class ModuleCompleter: diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4ad14..447eb1e551e774 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 560a9db192169e..572eee520e53f3 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -606,6 +606,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 9b6e80fdad9747..b7cd7940eb15b3 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -2929,12 +2929,6 @@ def make_clean_env() -> dict[str, str]: return clean_env -def initialized_with_pyrepl(): - """Detect whether PyREPL was used during Python initialization.""" - # If the main module has a __file__ attribute it's a Python module, which means PyREPL. - return hasattr(sys.modules["__main__"], "__file__") - - WINDOWS_STATUS = { 0xC0000005: "STATUS_ACCESS_VIOLATION", 0xC00000FD: "STATUS_STACK_OVERFLOW", diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 29762232d43b89..abb4bd1bc25fb1 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -926,6 +926,7 @@ def tearDown(self): def prepare_reader(self, events, namespace): console = FakeConsole(events) config = ReadlineConfig() + config.module_completer = ModuleCompleter(namespace) config.readline_completer = rlcompleter.Completer(namespace).complete reader = ReadlineAlikeReader(console=console, config=config) return reader @@ -1022,13 +1023,15 @@ def test_builtin_completion_top_level(self): def test_relative_import_completions(self): cases = ( - ("from .readl\t\n", "from .readline"), - ("from . import readl\t\n", "from . import readline"), + (None, "from .readl\t\n", "from .readl"), + (None, "from . import readl\t\n", "from . import readl"), + ("_pyrepl", "from .readl\t\n", "from .readline"), + ("_pyrepl", "from . import readl\t\n", "from . import readline"), ) - for code, expected in cases: + for package, code, expected in cases: with self.subTest(code=code): events = code_to_events(code) - reader = self.prepare_reader(events, namespace={}) + reader = self.prepare_reader(events, namespace={"__package__": package}) output = reader.readline() self.assertEqual(output, expected) @@ -1397,7 +1400,7 @@ def _assertMatchOK( ) @force_not_colorized - def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False, pythonstartup=False): clean_env = make_clean_env() clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses @@ -1406,9 +1409,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False blue.mkdir() mod = blue / "calx.py" mod.write_text("FOO = 42", encoding="utf-8") + startup = blue / "startup.py" + startup.write_text("BAR = 64", encoding="utf-8") commands = [ "print(f'^{" + var + "=}')" for var in expectations ] + ["exit()"] + if pythonstartup: + clean_env["PYTHONSTARTUP"] = str(startup) if as_file and as_module: self.fail("as_file and as_module are mutually exclusive") elif as_file: @@ -1427,7 +1434,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False skip=True, ) else: - self.fail("Choose one of as_file or as_module") + output, exit_code = self.run_repl( + commands, + cmdline_args=[], + env=clean_env, + cwd=td, + skip=True, + ) self.assertEqual(exit_code, 0) for var, expected in expectations.items(): @@ -1440,6 +1453,23 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) + def test_globals_initialized_as_default(self): + expectations = { + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations) + + def test_globals_initialized_from_pythonstartup(self): + expectations = { + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_file(self): expectations = { "FOO": "42", @@ -1449,6 +1479,16 @@ def test_inspect_keeps_globals_from_inspected_file(self): } self._run_repl_globals_test(expectations, as_file=True) + def test_inspect_keeps_globals_from_inspected_file_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_module(self): expectations = { "FOO": "42", @@ -1458,26 +1498,32 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) + def test_inspect_keeps_globals_from_inspected_module_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True, pythonstartup=True) + @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() - commands = ("from test.support import initialized_with_pyrepl\n" - "initialized_with_pyrepl()\n" - "exit()\n") - + pyrepl_commands = "clear\nexit()\n" env.pop("PYTHON_BASIC_REPL", None) - output, exit_code = self.run_repl(commands, env=env, skip=True) + output, exit_code = self.run_repl(pyrepl_commands, env=env, skip=True) self.assertEqual(exit_code, 0) - self.assertIn("True", output) - self.assertNotIn("False", output) self.assertNotIn("Exception", output) + self.assertNotIn("NameError", output) self.assertNotIn("Traceback", output) + basic_commands = "help\nexit()\n" env["PYTHON_BASIC_REPL"] = "1" - output, exit_code = self.run_repl(commands, env=env) + output, exit_code = self.run_repl(basic_commands, env=env) self.assertEqual(exit_code, 0) - self.assertIn("False", output) - self.assertNotIn("True", output) + self.assertIn("Type help() for interactive help", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst new file mode 100644 index 00000000000000..730d8a5af51f54 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst @@ -0,0 +1,3 @@ +PyREPL interactive shell no longer starts with ``__package__`` and +``__file__`` global names set to ``_pyrepl`` package internals. Contributed +by Yuichiro Tachibana. diff --git a/Modules/main.c b/Modules/main.c index ea1239ecc57f00..9b2ee103d52662 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -269,13 +269,14 @@ pymain_run_command(wchar_t *command) static int -pymain_start_pyrepl_no_main(void) +pymain_start_pyrepl(int pythonstartup) { int res = 0; PyObject *console = NULL; PyObject *empty_tuple = NULL; PyObject *kwargs = NULL; PyObject *console_result = NULL; + PyObject *main_module = NULL; PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); if (pyrepl == NULL) { @@ -299,7 +300,13 @@ pymain_start_pyrepl_no_main(void) res = pymain_exit_err_print(); goto done; } - if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { + main_module = PyImport_AddModuleRef("__main__"); + if (main_module == NULL) { + res = pymain_exit_err_print(); + goto done; + } + if (!PyDict_SetItemString(kwargs, "mainmodule", main_module) + && !PyDict_SetItemString(kwargs, "pythonstartup", pythonstartup ? Py_True : Py_False)) { console_result = PyObject_Call(console, empty_tuple, kwargs); if (console_result == NULL) { res = pymain_exit_err_print(); @@ -311,6 +318,7 @@ pymain_start_pyrepl_no_main(void) Py_XDECREF(empty_tuple); Py_XDECREF(console); Py_XDECREF(pyrepl); + Py_XDECREF(main_module); return res; } @@ -562,7 +570,7 @@ pymain_run_stdin(PyConfig *config) int run = PyRun_AnyFileExFlags(stdin, "", 0, &cf); return (run != 0); } - return pymain_run_module(L"_pyrepl", 0); + return pymain_start_pyrepl(0); } @@ -595,7 +603,7 @@ pymain_repl(PyConfig *config, int *exitcode) *exitcode = (run != 0); return; } - int run = pymain_start_pyrepl_no_main(); + int run = pymain_start_pyrepl(1); *exitcode = (run != 0); return; } From 4e9248f685225b517d528d4bd3c5682067f6b505 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 05:14:10 +0200 Subject: [PATCH 192/713] [3.14] gh-134062: Fix hash collisions in IPv4Network and IPv6Network (GH-134063) (#134476) gh-134062: Fix hash collisions in IPv4Network and IPv6Network (GH-134063) (cherry picked from commit f3fc0c16e08b317cb201cf1073e934e6909f1251) gh-134062: Fix hash collisions in IPv4Network and IPv6Network gh-134062: Add hash collision regression test Co-authored-by: Mike Salvatore --- Lib/ipaddress.py | 2 +- Lib/test/test_ipaddress.py | 28 +++++++++++++++++++ ...-05-15-14-27-01.gh-issue-134062.fRbJet.rst | 3 ++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 703fa289dda1fb..d8a84f33264dc5 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -729,7 +729,7 @@ def __eq__(self, other): return NotImplemented def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) + return hash((int(self.network_address), int(self.netmask))) def __contains__(self, other): # always false if one is v4 and the other is v6. diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index d04012d1afd540..a06608d0016253 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -2762,6 +2762,34 @@ def testV6HashIsNotConstant(self): ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV4HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.255/32").__hash__(), + ipaddress.IPv4Network("192.168.1.0/24").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("172.24.255.0/24").__hash__(), + ipaddress.IPv4Network("172.24.0.0/16").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.87/32").__hash__(), + ipaddress.IPv4Network("192.168.1.86/31").__hash__() + ) + + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV6HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv6Network("fe80::/64").__hash__(), + ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("10.0.0.0/8").__hash__(), + ipaddress.IPv6Network( + "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112" + ).__hash__() + ) + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst new file mode 100644 index 00000000000000..f62a3ec480193d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst @@ -0,0 +1,3 @@ +:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for +:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` +objects. From 03a211290e86cd276c71e28097aa868984e28b51 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 06:15:06 +0200 Subject: [PATCH 193/713] [3.14] gh-134309: Add ``github.actor`` to the GitHub Actions concurrency key (GH-134310) (#134484) gh-134309: Add ``github.actor`` to the GitHub Actions concurrency key (GH-134310) When inexperienced users create a PR from their default branch, all of the concurrency keys collide as there is no namespacing. This becomes an issue at events with many new contributors, where workflow runs are cancelled on other pull requests. Disambiguate by adding the username of the relevant 'actor' to the concurrency key. (cherry picked from commit 979d81a17905e922d32fb1671f9ed394e0ffbda6) Authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Kira Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> Co-authored-by: Sviatoslav Sydorenko Co-authored-by: C.A.M. Gerlach --- .github/workflows/build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3478af18f0e9cb..e26433786f74e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,13 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency + # 'group' must be a key uniquely representing a PR or push event. + # github.workflow is the workflow name + # github.actor is the user invoking the workflow + # github.head_ref is the source branch of the PR or otherwise blank + # github.run_id is a unique number for the current run + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: From 9cef5c1f65d884f73f5ed1d7191a5621232e17b0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 10:20:11 +0200 Subject: [PATCH 194/713] [3.14] Add notes on nogil & reinitialization to the Opt-Out section in Module Isolation HOWTO (GH-134141) (GH-134492) Add notes on nogil & reinitialization to the Opt-Out section in Module Isolation HOWTO (GH-134141) (cherry picked from commit 1f0a294e8c2ff009c6b74ca5aa71da6269aec0dd) Co-authored-by: Petr Viktorin Co-authored-by: Sam Gross --- Doc/howto/isolating-extensions.rst | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index a636e06bda8344..5513cd7367519f 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -215,21 +215,36 @@ multiple interpreters correctly. If this is not yet the case for your module, you can explicitly make your module loadable only once per process. For example:: + // A process-wide flag static int loaded = 0; + // Mutex to provide thread safety (only needed for free-threaded Python) + static PyMutex modinit_mutex = {0}; + static int exec_module(PyObject* module) { + PyMutex_Lock(&modinit_mutex); if (loaded) { + PyMutex_Unlock(&modinit_mutex); PyErr_SetString(PyExc_ImportError, "cannot load module more than once per process"); return -1; } loaded = 1; + PyMutex_Unlock(&modinit_mutex); // ... rest of initialization } +If your module's :c:member:`PyModuleDef.m_clear` function is able to prepare +for future re-initialization, it should clear the ``loaded`` flag. +In this case, your module won't support multiple instances existing +*concurrently*, but it will, for example, support being loaded after +Python runtime shutdown (:c:func:`Py_FinalizeEx`) and re-initialization +(:c:func:`Py_Initialize`). + + Module State Access from Functions ---------------------------------- From 64c265a63805fdc9eac62ac853891c8a58033201 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 10:34:46 +0200 Subject: [PATCH 195/713] [3.14] gh-131357: Add some extra tests for empty bytes and bytearray (GH-134458) (#134489) gh-131357: Add some extra tests for empty bytes and bytearray (GH-134458) (cherry picked from commit 7309eb60c077836b4b05a86ed23c710dbff737ec) Co-authored-by: Sergey Miryanov --- Lib/test/test_capi/test_bytearray.py | 6 ++++++ Lib/test/test_capi/test_bytes.py | 3 +++ 2 files changed, 9 insertions(+) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index dfa98de9f007d8..52565ea34c61b8 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -66,6 +66,7 @@ def test_fromobject(self): # Test PyByteArray_FromObject() fromobject = _testlimitedcapi.bytearray_fromobject + self.assertEqual(fromobject(b''), bytearray(b'')) self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) @@ -115,6 +116,7 @@ def test_concat(self): self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) + self.assertEqual(concat(bytearray(b''), bytearray(b'')), bytearray(b'')) self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), bytearray(b'abcdef')) self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), @@ -150,6 +152,10 @@ def test_resize(self): self.assertEqual(resize(ba, 0), 0) self.assertEqual(ba, bytearray()) + ba = bytearray(b'') + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + ba = ByteArraySubclass(b'abcdef') self.assertEqual(resize(ba, 3), 0) self.assertEqual(ba, bytearray(b'abc')) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 86d5692893fc98..bc820bd68d9e21 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -81,6 +81,7 @@ def test_fromobject(self): # Test PyBytes_FromObject() fromobject = _testlimitedcapi.bytes_fromobject + self.assertEqual(fromobject(b''), b'') self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') @@ -138,6 +139,7 @@ def test_repr(self): # Test PyBytes_Repr() bytes_repr = _testlimitedcapi.bytes_repr + self.assertEqual(bytes_repr(b'', 0), r"""b''""") self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") @@ -197,6 +199,7 @@ def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" decodeescape = _testlimitedcapi.bytes_decodeescape + self.assertEqual(decodeescape(b''), b'') self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), b'''\t\n\r\v\f\0\\'"''') From d5f7e80d4407655919b32bbd5746b23af174bc5b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 11:57:42 +0200 Subject: [PATCH 196/713] [3.14] Consistent sentence case in docs template files (GH-134412) (#134495) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/tools/templates/customsourcelink.html | 6 +++--- Doc/tools/templates/download.html | 2 +- Doc/tools/templates/indexcontent.html | 2 +- Doc/tools/templates/indexsidebar.html | 10 +++++----- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index eb9db9e341bd75..43d3a7a892a880 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -1,11 +1,11 @@ {%- if show_source and has_source and sourcename %}
-

{{ _('This Page') }}

+

{{ _('This page') }}

diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html index 4645f7d394e29e..47a57eb111ba50 100644 --- a/Doc/tools/templates/download.html +++ b/Doc/tools/templates/download.html @@ -27,7 +27,7 @@ {%- endblock -%} {% block body %} -

{% trans %}Download Python {{ dl_version }} Documentation{% endtrans %}

+

{% trans %}Download Python {{ dl_version }} documentation{% endtrans %}

{% if last_updated %}

{% trans %}Last updated on: {{ last_updated }}.{% endtrans %}

{% endif %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html index 06a4223643a05a..544cc4234f441e 100644 --- a/Doc/tools/templates/indexcontent.html +++ b/Doc/tools/templates/indexcontent.html @@ -72,7 +72,7 @@

{{ docstitle|e }}

{% trans %}Reporting issues{% endtrans %}

-

{% trans %}Contributing to Docs{% endtrans %}

+

{% trans %}Contributing to docs{% endtrans %}

{% trans %}Download the documentation{% endtrans %}

{% trans %}History and license of Python{% endtrans %}

diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index eea29e2449a9cf..086f15662cf87b 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -9,9 +9,9 @@

{% trans %}Docs by version{% endtrans %}

{% trans %}Other resources{% endtrans %}

From db98e0bb127c469842dc119d23e5daacd23862cc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 12:42:50 +0200 Subject: [PATCH 197/713] [3.14] gh-71339: Use new assertion methods in tests (GH-129046) (GH-134498) (cherry picked from commit 2602d8ae981c4bae1cada2c174b367d97f712efb) Co-authored-by: Serhiy Storchaka --- Lib/test/_test_embed_structseq.py | 2 +- Lib/test/datetimetester.py | 12 ++++---- Lib/test/mapping_tests.py | 4 +-- Lib/test/pickletester.py | 4 +-- Lib/test/support/warnings_helper.py | 3 +- Lib/test/test__osx_support.py | 4 +-- Lib/test/test_abstract_numbers.py | 30 ++++++++++---------- Lib/test/test_argparse.py | 2 +- Lib/test/test_ast/test_ast.py | 18 ++++++------ Lib/test/test_audit.py | 2 +- Lib/test/test_base64.py | 2 +- Lib/test/test_baseexception.py | 8 ++---- Lib/test/test_binascii.py | 6 ++-- Lib/test/test_binop.py | 2 +- Lib/test/test_buffer.py | 4 +-- Lib/test/test_builtin.py | 4 +-- Lib/test/test_bytes.py | 10 +++---- Lib/test/test_bz2.py | 4 +-- Lib/test/test_calendar.py | 2 +- Lib/test/test_call.py | 4 +-- Lib/test/test_clinic.py | 23 ++++++--------- Lib/test/test_cmd_line.py | 11 +++---- Lib/test/test_cmd_line_script.py | 11 ++++--- Lib/test/test_code_module.py | 2 +- Lib/test/test_codecs.py | 2 +- Lib/test/test_compileall.py | 2 +- Lib/test/test_compiler_assemble.py | 2 +- Lib/test/test_contextlib.py | 8 +++--- Lib/test/test_contextlib_async.py | 8 +++--- Lib/test/test_copy.py | 2 +- Lib/test/test_coroutines.py | 2 +- Lib/test/test_curses.py | 2 +- Lib/test/test_dataclasses/__init__.py | 24 ++++++++-------- Lib/test/test_dbm.py | 2 +- Lib/test/test_dbm_sqlite3.py | 4 +-- Lib/test/test_deque.py | 2 +- Lib/test/test_descr.py | 18 ++++++------ Lib/test/test_dict.py | 4 +-- Lib/test/test_dynamicclassattribute.py | 4 +-- Lib/test/test_enum.py | 18 ++++++------ Lib/test/test_errno.py | 6 ++-- Lib/test/test_exception_group.py | 10 +++---- Lib/test/test_exceptions.py | 20 ++++++------- Lib/test/test_fileinput.py | 2 +- Lib/test/test_gc.py | 2 +- Lib/test/test_genericalias.py | 10 +++---- Lib/test/test_genericpath.py | 4 +-- Lib/test/test_gzip.py | 8 +++--- Lib/test/test_hashlib.py | 4 +-- Lib/test/test_inspect/test_inspect.py | 10 +++---- Lib/test/test_int.py | 2 +- Lib/test/test_io.py | 18 ++++++------ Lib/test/test_json/test_fail.py | 2 +- Lib/test/test_json/test_tool.py | 2 +- Lib/test/test_launcher.py | 8 +++--- Lib/test/test_lzma.py | 4 +-- Lib/test/test_memoryio.py | 4 +-- Lib/test/test_ordered_dict.py | 8 +++--- Lib/test/test_os.py | 14 ++++----- Lib/test/test_pathlib/test_pathlib.py | 8 +++--- Lib/test/test_peepholer.py | 2 +- Lib/test/test_peg_generator/test_c_parser.py | 4 +-- Lib/test/test_peg_generator/test_pegen.py | 6 ++-- Lib/test/test_perf_profiler.py | 4 +-- Lib/test/test_pickle.py | 4 +-- Lib/test/test_platform.py | 2 +- Lib/test/test_posix.py | 12 ++++---- Lib/test/test_property.py | 4 +-- Lib/test/test_pulldom.py | 4 +-- Lib/test/test_pyclbr.py | 2 +- Lib/test/test_pydoc/test_pydoc.py | 2 +- Lib/test/test_random.py | 12 ++++---- Lib/test/test_re.py | 8 +++--- Lib/test/test_reprlib.py | 20 ++++++------- Lib/test/test_rlcompleter.py | 2 +- Lib/test/test_runpy.py | 2 +- Lib/test/test_scope.py | 2 +- Lib/test/test_script_helper.py | 3 +- Lib/test/test_set.py | 6 ++-- Lib/test/test_shutil.py | 6 ++-- Lib/test/test_site.py | 22 +++++++------- Lib/test/test_socket.py | 22 +++++++------- Lib/test/test_source_encoding.py | 3 +- Lib/test/test_ssl.py | 10 +++---- Lib/test/test_stat.py | 6 ++-- Lib/test/test_statistics.py | 21 ++++---------- Lib/test/test_structseq.py | 4 +-- Lib/test/test_subprocess.py | 8 +++--- Lib/test/test_super.py | 10 +++---- Lib/test/test_support.py | 4 +-- Lib/test/test_sys.py | 26 ++++++++--------- Lib/test/test_sysconfig.py | 10 +++---- Lib/test/test_tarfile.py | 2 +- Lib/test/test_tempfile.py | 16 +++++------ Lib/test/test_termios.py | 4 +-- Lib/test/test_time.py | 12 ++++---- Lib/test/test_timeit.py | 4 +-- Lib/test/test_tkinter/support.py | 2 +- Lib/test/test_tkinter/test_misc.py | 24 ++++++++-------- Lib/test/test_type_comments.py | 2 +- Lib/test/test_types.py | 22 +++++++------- Lib/test/test_unittest/test_case.py | 8 +++--- Lib/test/test_userdict.py | 2 +- Lib/test/test_venv.py | 4 +-- Lib/test/test_warnings/__init__.py | 10 +++---- Lib/test/test_weakref.py | 6 ++-- Lib/test/test_weakset.py | 4 +-- Lib/test/test_winconsoleio.py | 6 ++-- Lib/test/test_with.py | 2 +- Lib/test/test_wmi.py | 4 +-- Lib/test/test_wsgiref.py | 14 ++++----- Lib/test/test_xml_etree.py | 5 ++-- Lib/test/test_xxlimited.py | 2 +- Lib/test/test_zipapp.py | 6 ++-- Lib/test/test_zipfile/test_core.py | 6 ++-- Lib/test/test_zipimport.py | 4 +-- Lib/test/test_zoneinfo/test_zoneinfo.py | 4 +-- 117 files changed, 407 insertions(+), 445 deletions(-) diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 154662efce9412..4cac84d7a469ac 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -11,7 +11,7 @@ def check_structseq(self, obj_type): # ob_refcnt self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - self.assertTrue(issubclass(obj_type, tuple)) + self.assertIsSubclass(obj_type, tuple) # tp_bases self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index d1882a310bbbb0..1b551254f86c32 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -183,7 +183,7 @@ class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name - self.assertTrue(issubclass(NotEnough, tzinfo)) + self.assertIsSubclass(NotEnough, tzinfo) ne = NotEnough(3, "NotByALongShot") self.assertIsInstance(ne, tzinfo) @@ -232,7 +232,7 @@ def test_pickling_subclass(self): self.assertIs(type(derived), otype) self.assertEqual(derived.utcoffset(None), offset) self.assertEqual(derived.tzname(None), oname) - self.assertFalse(hasattr(derived, 'spam')) + self.assertNotHasAttr(derived, 'spam') def test_issue23600(self): DSTDIFF = DSTOFFSET = timedelta(hours=1) @@ -813,7 +813,7 @@ def test_roundtrip(self): # Verify td -> string -> td identity. s = repr(td) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] td2 = eval(s) self.assertEqual(td, td2) @@ -1231,7 +1231,7 @@ def test_roundtrip(self): self.theclass.today()): # Verify dt -> string -> date identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -2218,7 +2218,7 @@ def test_roundtrip(self): self.theclass.now()): # Verify dt -> string -> datetime identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -3672,7 +3672,7 @@ def test_roundtrip(self): # Verify t -> string -> time identity. s = repr(t) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] t2 = eval(s) self.assertEqual(t, t2) diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index 9d38da5a86ee5a..20306e1526d7b8 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -70,8 +70,8 @@ def test_read(self): if not d: self.fail("Full mapping must compare to True") # keys(), items(), iterkeys() ... def check_iterandlist(iter, lst, ref): - self.assertTrue(hasattr(iter, '__next__')) - self.assertTrue(hasattr(iter, '__iter__')) + self.assertHasAttr(iter, '__next__') + self.assertHasAttr(iter, '__iter__') x = list(iter) self.assertTrue(set(x)==set(lst)==set(ref)) check_iterandlist(iter(d.keys()), list(d.keys()), diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index dcba6369541049..9d6ae3e4d00ece 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -3068,7 +3068,7 @@ def test_proto(self): pickled = self.dumps(None, proto) if proto >= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) @@ -5007,7 +5007,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index a6e43dff2003b7..5f6f14afd74a6e 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -23,8 +23,7 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(len(warns), 1, warns) warn, = warns - testcase.assertTrue(issubclass(warn.category, SyntaxWarning), - warn.category) + testcase.assertIsSubclass(warn.category, SyntaxWarning) if errtext: testcase.assertRegex(str(warn.message), errtext) testcase.assertEqual(warn.filename, '') diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 53aa26620a6475..0813c4804c1cdc 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -66,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 72232b670cdb89..cf071d2c933dd2 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -24,11 +24,11 @@ def not_implemented(*args, **kwargs): class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Rational)) - self.assertTrue(issubclass(int, Real)) - self.assertTrue(issubclass(int, Complex)) - self.assertTrue(issubclass(int, Number)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -38,11 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Integral)) - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) - self.assertTrue(issubclass(float, Complex)) - self.assertTrue(issubclass(float, Number)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -50,11 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Integral)) - self.assertFalse(issubclass(complex, Rational)) - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) - self.assertTrue(issubclass(complex, Number)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 58853ba4eb3674..08ff41368d9bb0 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6805,7 +6805,7 @@ class TestImportStar(TestCase): def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 0776559b9003db..1479a8eafd62ec 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -275,12 +275,12 @@ def test_alias(self): self.assertEqual(alias.end_col_offset, 17) def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) def test_field_attr_existence(self): for name, item in ast.__dict__.items(): @@ -1101,7 +1101,7 @@ def test_copy_with_parents(self): def test_replace_interface(self): for klass in self.iter_ast_classes(): with self.subTest(klass=klass): - self.assertTrue(hasattr(klass, '__replace__')) + self.assertHasAttr(klass, '__replace__') fields = set(klass._fields) with self.subTest(klass=klass, fields=fields): @@ -1330,7 +1330,7 @@ def test_replace_reject_known_custom_instance_fields_commits(self): context = node.ctx # explicit rejection of known instance fields - self.assertTrue(hasattr(node, 'extra')) + self.assertHasAttr(node, 'extra') msg = "Name.__replace__ got an unexpected keyword argument 'extra'." with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node, extra=1) @@ -3071,7 +3071,7 @@ def test_FunctionDef(self): with self.assertWarnsRegex(DeprecationWarning, r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): node = ast.FunctionDef(args=args) - self.assertFalse(hasattr(node, "name")) + self.assertNotHasAttr(node, "name") self.assertEqual(node.decorator_list, []) node = ast.FunctionDef(name='foo', args=args) self.assertEqual(node.name, 'foo') diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 2b24b5d79275fa..5f9eb381f605d9 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -134,7 +134,7 @@ def test_socket(self): self.assertEqual(events[0][0], "socket.gethostname") self.assertEqual(events[1][0], "socket.__new__") self.assertEqual(events[2][0], "socket.bind") - self.assertTrue(events[2][2].endswith("('127.0.0.1', 8080)")) + self.assertEndsWith(events[2][2], "('127.0.0.1', 8080)") def test_gc(self): returncode, events, stderr = self.run_python("test_gc") diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 9efebc43d911c4..ce2e3e3726fcd0 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -812,7 +812,7 @@ def test_decode_nonascii_str(self): self.assertRaises(ValueError, f, 'with non-ascii \xcb') def test_ErrorHeritage(self): - self.assertTrue(issubclass(binascii.Error, ValueError)) + self.assertIsSubclass(binascii.Error, ValueError) def test_RFC4648_test_cases(self): # test cases from RFC 4648 section 10 diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index e599b02c17d9c0..12d4088842b119 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,13 +10,11 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation @@ -65,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 1f3b6746ce4a62..7ed7d7c47b6de1 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -38,13 +38,13 @@ def assertConversion(self, original, converted, restored, **kwargs): def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) def test_returned_value(self): diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c4983df..b224c3d4e6078e 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 61921e93e85e63..19582e757161fc 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -2879,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith("Calendar for 2004', output) def test_html_output_current_year(self): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 185ae84dc4d19f..1c73aaafb71fd5 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -695,8 +695,8 @@ class DerivedType(SuperType): UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) # Aside: Quickly check that the C helper actually made derived types - self.assertTrue(issubclass(UnaffectedType1, DerivedType)) - self.assertTrue(issubclass(UnaffectedType2, SuperType)) + self.assertIsSubclass(UnaffectedType1, DerivedType) + self.assertIsSubclass(UnaffectedType2, SuperType) # Initial state: tp_call self.assertEqual(instance(), "tp_call") diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 90b7d343cea583..05e46a18b0a92a 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -238,11 +238,11 @@ def test_directive_output_print(self): # The generated output will differ for every run, but we can check that # it starts with the clinic block, we check that it contains all the # expected fields, and we check that it contains the checksum line. - self.assertTrue(out.startswith(dedent(""" + self.assertStartsWith(out, dedent(""" /*[clinic input] output print 'I told you once.' [clinic start generated code]*/ - """))) + """)) fields = { "cpp_endif", "cpp_if", @@ -259,9 +259,7 @@ def test_directive_output_print(self): with self.subTest(field=field): self.assertIn(field, out) last_line = out.rstrip().split("\n")[-1] - self.assertTrue( - last_line.startswith("/*[clinic end generated code: output=") - ) + self.assertStartsWith(last_line, "/*[clinic end generated code: output=") def test_directive_wrong_arg_number(self): raw = dedent(""" @@ -2705,8 +2703,7 @@ def test_cli_force(self): # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. _, err = self.expect_failure(fn) - self.assertTrue(err.endswith(fail_msg), - f"{err!r} does not end with {fail_msg!r}") + self.assertEndsWith(err, fail_msg) # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -2717,8 +2714,7 @@ def test_cli_force(self): ) with open(fn, encoding='utf-8') as f: generated = f.read() - self.assertTrue(generated.endswith(checksum), - (generated, checksum)) + self.assertEndsWith(generated, checksum) def test_cli_make(self): c_code = dedent(""" @@ -2867,8 +2863,8 @@ def test_cli_converters(self): # param may change (it's a set, thus unordered). So, let's compare the # start and end of the expected output, and then assert that the # converters appear lined up in alphabetical order. - self.assertTrue(out.startswith(prelude), out) - self.assertTrue(out.endswith(finale), out) + self.assertStartsWith(out, prelude) + self.assertEndsWith(out, finale) out = out.removeprefix(prelude) out = out.removesuffix(finale) @@ -2876,10 +2872,7 @@ def test_cli_converters(self): for converter, line in zip(expected_converters, lines): line = line.lstrip() with self.subTest(converter=converter): - self.assertTrue( - line.startswith(converter), - f"expected converter {converter!r}, got {line!r}" - ) + self.assertStartsWith(line, converter) def test_cli_fail_converters_and_filename(self): _, err = self.expect_failure("--converters", "test.c") diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index f540973c11e01c..c17d749d4a17ed 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -39,7 +39,8 @@ def test_directories(self): def verify_valid_flag(self, cmd_line): rc, out, err = assert_python_ok(cmd_line) - self.assertTrue(out == b'' or out.endswith(b'\n')) + if out != b'': + self.assertEndsWith(out, b'\n') self.assertNotIn(b'Traceback', out) self.assertNotIn(b'Traceback', err) return out @@ -89,8 +90,8 @@ def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': rc, out, err = assert_python_ok(switch) - self.assertFalse(err.startswith(version)) - self.assertTrue(out.startswith(version)) + self.assertNotStartsWith(err, version) + self.assertStartsWith(out, version) def test_verbose(self): # -v causes imports to write to stderr. If the write to @@ -380,7 +381,7 @@ def test_unbuffered_input(self): p.stdin.flush() data, rc = _kill_python_and_exit_code(p) self.assertEqual(rc, 0) - self.assertTrue(data.startswith(b'x'), data) + self.assertStartsWith(data, b'x') def test_large_PYTHONPATH(self): path1 = "ABCDE" * 100 @@ -1039,7 +1040,7 @@ def test_parsing_error(self): stderr=subprocess.PIPE, text=True) err_msg = "Unknown option: --unknown-option\nusage: " - self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) + self.assertStartsWith(proc.stderr, err_msg) self.assertNotEqual(proc.returncode, 0) def test_int_max_str_digits(self): diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 53dc9b1a7effb5..784c45aa96f8a7 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -553,9 +553,9 @@ def test_pep_409_verbiage(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = stderr.decode('ascii').split('\n') self.assertEqual(len(text), 5) - self.assertTrue(text[0].startswith('Traceback')) - self.assertTrue(text[1].startswith(' File ')) - self.assertTrue(text[3].startswith('NameError')) + self.assertStartsWith(text[0], 'Traceback') + self.assertStartsWith(text[1], ' File ') + self.assertStartsWith(text[3], 'NameError') def test_non_ascii(self): # Apple platforms deny the creation of a file with an invalid UTF-8 name. @@ -708,9 +708,8 @@ def test_syntaxerror_does_not_crash(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() # It used to crash in https://github.com/python/cpython/issues/111132 - self.assertTrue(text.endswith( - 'SyntaxError: nonlocal declaration not allowed at module level\n', - ), text) + self.assertEndsWith(text, + 'SyntaxError: nonlocal declaration not allowed at module level\n') def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 57fb130070b34e..3642b47c2c1f03 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -133,7 +133,7 @@ def test_unicode_error(self): output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] output = output[output.index('\n') + 1:] - self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertStartsWith(output, 'UnicodeEncodeError: ') self.assertIs(self.sysmod.last_type, UnicodeEncodeError) self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) self.assertIsNone(self.sysmod.last_traceback) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 1d6136405f428c..8c9a0972492294 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -3802,7 +3802,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a580a240d9f474..8384c183dd92dd 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -316,7 +316,7 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertTrue(mods) for mod in mods: - self.assertTrue(mod.startswith(self.directory), mod) + self.assertStartsWith(mod, self.directory) modcode = importlib.util.cache_from_source(mod) modpath = mod[len(self.directory+os.sep):] _, _, err = script_helper.assert_python_failure(modcode) diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index c4962e3599986e..99a11e99d56485 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -146,4 +146,4 @@ def test_exception_table(self): L1 to L2 -> L2 [0] L2 to L3 -> L3 [1] lasti """) - self.assertTrue(output.getvalue().endswith(exc_table)) + self.assertEndsWith(output.getvalue(), exc_table) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index cf6519598037e9..6a3329fa5aaace 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -48,23 +48,23 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractContextManager) class DefaultEnter(AbstractContextManager): def __exit__(self, *args): super().__exit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractContextManager) class NoEnter(ManagerFromScratch): __enter__ = None - self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + self.assertNotIsSubclass(NoEnter, AbstractContextManager) class NoExit(ManagerFromScratch): __exit__ = None - self.assertFalse(issubclass(NoExit, AbstractContextManager)) + self.assertNotIsSubclass(NoExit, AbstractContextManager) class ContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 7750186e56a5cc..dcd0072037950e 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -77,23 +77,23 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractAsyncContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractAsyncContextManager) class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): await super().__aexit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractAsyncContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractAsyncContextManager) class NoneAenter(ManagerFromScratch): __aenter__ = None - self.assertFalse(issubclass(NoneAenter, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAenter, AbstractAsyncContextManager) class NoneAexit(ManagerFromScratch): __aexit__ = None - self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAexit, AbstractAsyncContextManager) class AsyncContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index d76341417e9bef..3cb8072abefe98 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -19,7 +19,7 @@ class TestCopy(unittest.TestCase): def test_exceptions(self): self.assertIs(copy.Error, copy.error) - self.assertTrue(issubclass(copy.Error, Exception)) + self.assertIsSubclass(copy.Error, Exception) # The copy() method diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 761cb230277bd9..4755046fe1904d 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -527,7 +527,7 @@ class CoroutineTest(unittest.TestCase): def test_gen_1(self): def gen(): yield - self.assertFalse(hasattr(gen, '__await__')) + self.assertNotHasAttr(gen, '__await__') def test_func_1(self): async def foo(): diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index c307258e5657e2..181ead710f3d68 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -1257,7 +1257,7 @@ class TestAscii(unittest.TestCase): def test_controlnames(self): for name in curses.ascii.controlnames: - self.assertTrue(hasattr(curses.ascii, name), name) + self.assertHasAttr(curses.ascii, name) def test_ctypes(self): def check(func, expected): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ac78f8327b808e..e98a8f284cec9f 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -120,7 +120,7 @@ class Some: pass for param in inspect.signature(dataclass).parameters: if param == 'cls': continue - self.assertTrue(hasattr(Some.__dataclass_params__, param), msg=param) + self.assertHasAttr(Some.__dataclass_params__, param) def test_named_init_params(self): @dataclass @@ -671,7 +671,7 @@ class C: self.assertEqual(the_fields[0].name, 'x') self.assertEqual(the_fields[0].type, int) - self.assertFalse(hasattr(C, 'x')) + self.assertNotHasAttr(C, 'x') self.assertTrue (the_fields[0].init) self.assertTrue (the_fields[0].repr) self.assertEqual(the_fields[1].name, 'y') @@ -681,7 +681,7 @@ class C: self.assertTrue (the_fields[1].repr) self.assertEqual(the_fields[2].name, 'z') self.assertEqual(the_fields[2].type, str) - self.assertFalse(hasattr(C, 'z')) + self.assertNotHasAttr(C, 'z') self.assertTrue (the_fields[2].init) self.assertFalse(the_fields[2].repr) @@ -732,8 +732,8 @@ class C: z: object = default t: int = field(default=100) - self.assertFalse(hasattr(C, 'x')) - self.assertFalse(hasattr(C, 'y')) + self.assertNotHasAttr(C, 'x') + self.assertNotHasAttr(C, 'y') self.assertIs (C.z, default) self.assertEqual(C.t, 100) @@ -2912,10 +2912,10 @@ class C: pass c = C() - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): c.i = 5 - self.assertFalse(hasattr(c, 'i')) + self.assertNotHasAttr(c, 'i') with self.assertRaises(FrozenInstanceError): del c.i @@ -3144,7 +3144,7 @@ class S(D): del s.y self.assertEqual(s.y, 10) del s.cached - self.assertFalse(hasattr(s, 'cached')) + self.assertNotHasAttr(s, 'cached') with self.assertRaises(AttributeError) as cm: del s.cached self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3158,12 +3158,12 @@ class S(D): pass s = S() - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') s.x = 5 self.assertEqual(s.x, 5) del s.x - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') with self.assertRaises(AttributeError) as cm: del s.x self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3393,8 +3393,8 @@ class A: B = dataclass(A, slots=True) self.assertIsNot(A, B) - self.assertFalse(hasattr(A, "__slots__")) - self.assertTrue(hasattr(B, "__slots__")) + self.assertNotHasAttr(A, "__slots__") + self.assertHasAttr(B, "__slots__") # Can't be local to test_frozen_pickle. @dataclass(frozen=True, slots=True) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 4be7c5649da68a..a10922a403ec38 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -66,7 +66,7 @@ def keys_helper(self, f): return keys def test_error(self): - self.assertTrue(issubclass(self.module.error, OSError)) + self.assertIsSubclass(self.module.error, OSError) def test_anydbm_not_existing(self): self.assertRaises(dbm.error, dbm.open, _fname) diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 2e1f2d32924bad..9216da8a63f957 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -36,7 +36,7 @@ def test_uri_substitutions(self): ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): @@ -55,7 +55,7 @@ def test_uri_windows(self): with self.subTest(path=path, normalized=normalized): if not Path(path).is_absolute(): self.skipTest(f"skipping relative path: {path!r}") - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) class ReadOnly(_SQLiteDbmTests): diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 4679f297fd7f4a..4e1a489205a685 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -838,7 +838,7 @@ def test_copy_pickle(self): self.assertEqual(list(d), list(e)) self.assertEqual(e.x, d.x) self.assertEqual(e.z, d.z) - self.assertFalse(hasattr(e, 'y')) + self.assertNotHasAttr(e, 'y') def test_pickle_recursive(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 14026531e22333..d635855f537af0 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -409,7 +409,7 @@ class ClassPropertiesAndMethods(unittest.TestCase): def test_python_dicts(self): # Testing Python subclass of dict... - self.assertTrue(issubclass(dict, dict)) + self.assertIsSubclass(dict, dict) self.assertIsInstance({}, dict) d = dict() self.assertEqual(d, {}) @@ -433,7 +433,7 @@ def setstate(self, state): self.state = state def getstate(self): return self.state - self.assertTrue(issubclass(C, dict)) + self.assertIsSubclass(C, dict) a1 = C(12) self.assertEqual(a1.state, 12) a2 = C(foo=1, bar=2) @@ -1048,15 +1048,15 @@ class SubType(types.ModuleType): m = types.ModuleType("m") self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") m.__class__ = SubType self.assertTrue(m.__class__ is SubType) - self.assertTrue(hasattr(m, "a")) + self.assertHasAttr(m, "a") m.__class__ = types.ModuleType self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") # Make sure that builtin immutable objects don't support __class__ # assignment, because the object instances may be interned. @@ -1780,7 +1780,7 @@ class D(C): class E: # *not* subclassing from C foo = C.foo self.assertEqual(E().foo.__func__, C.foo) # i.e., unbound - self.assertTrue(repr(C.foo.__get__(C())).startswith("", report) else: self.assertIn("test message", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") @cpython_only # Python built with Py_TRACE_REFS fail with a fatal error in diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b340ef7ed1621c..6524baabe7f96f 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -245,7 +245,7 @@ def test_detached_stdin_binary_mode(self): orig_stdin = sys.stdin try: sys.stdin = BytesIO(b'spam, bacon, sausage, and spam') - self.assertFalse(hasattr(sys.stdin, 'buffer')) + self.assertNotHasAttr(sys.stdin, 'buffer') fi = FileInput(files=['-'], mode='rb') lines = list(fi) self.assertEqual(lines, [b'spam, bacon, sausage, and spam']) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 95c98c6ac63f71..b4cbfb6d774080 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -914,7 +914,7 @@ def __del__(self): gc.collect() self.assertEqual(len(Lazarus.resurrected_instances), 1) instance = Lazarus.resurrected_instances.pop() - self.assertTrue(hasattr(instance, "cargo")) + self.assertHasAttr(instance, "cargo") self.assertEqual(id(instance.cargo), cargo_id) gc.collect() diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index ea0dc241e39475..7601cb00ff6685 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -236,13 +236,13 @@ class MyGeneric: self.assertEqual(repr(x2), 'tuple[*tuple[int, str]]') x3 = tuple[*tuple[int, ...]] self.assertEqual(repr(x3), 'tuple[*tuple[int, ...]]') - self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) + self.assertEndsWith(repr(MyList[int]), '.BaseTest.test_repr..MyList[int]') self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr # gh-105488 - self.assertTrue(repr(MyGeneric[int]).endswith('MyGeneric[int]')) - self.assertTrue(repr(MyGeneric[[]]).endswith('MyGeneric[[]]')) - self.assertTrue(repr(MyGeneric[[int, str]]).endswith('MyGeneric[[int, str]]')) + self.assertEndsWith(repr(MyGeneric[int]), 'MyGeneric[int]') + self.assertEndsWith(repr(MyGeneric[[]]), 'MyGeneric[[]]') + self.assertEndsWith(repr(MyGeneric[[int, str]]), 'MyGeneric[[int, str]]') def test_exposed_type(self): import types @@ -362,7 +362,7 @@ def test_isinstance(self): def test_issubclass(self): class L(list): ... - self.assertTrue(issubclass(L, list)) + self.assertIsSubclass(L, list) with self.assertRaises(TypeError): issubclass(L, list[str]) diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 6c3abe602f557c..df07af01fc7540 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -92,8 +92,8 @@ def test_commonprefix(self): for s1 in testlist: for s2 in testlist: p = commonprefix([s1, s2]) - self.assertTrue(s1.startswith(p)) - self.assertTrue(s2.startswith(p)) + self.assertStartsWith(s1, p) + self.assertStartsWith(s2, p) if s1 != s2: n = len(p) self.assertNotEqual(s1[n:n+1], s2[n:n+1]) diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index fa5de7c190e6a3..ccbacc7c19b6e6 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -331,13 +331,13 @@ def test_mode(self): def test_1647484(self): for mode in ('wb', 'rb'): with gzip.GzipFile(self.filename, mode) as f: - self.assertTrue(hasattr(f, "name")) + self.assertHasAttr(f, "name") self.assertEqual(f.name, self.filename) def test_paddedfile_getattr(self): self.test_write() with gzip.GzipFile(self.filename, 'rb') as f: - self.assertTrue(hasattr(f.fileobj, "name")) + self.assertHasAttr(f.fileobj, "name") self.assertEqual(f.fileobj.name, self.filename) def test_mtime(self): @@ -345,7 +345,7 @@ def test_mtime(self): with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite: fWrite.write(data1) with gzip.GzipFile(self.filename) as fRead: - self.assertTrue(hasattr(fRead, 'mtime')) + self.assertHasAttr(fRead, 'mtime') self.assertIsNone(fRead.mtime) dataRead = fRead.read() self.assertEqual(dataRead, data1) @@ -460,7 +460,7 @@ def test_zero_padded_file(self): self.assertEqual(d, data1 * 50, "Incorrect data in file") def test_gzip_BadGzipFile_exception(self): - self.assertTrue(issubclass(gzip.BadGzipFile, OSError)) + self.assertIsSubclass(gzip.BadGzipFile, OSError) def test_bad_gzip_file(self): with open(self.filename, 'wb') as file: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5e3356a02f31b6..de4c8a1670f591 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -152,8 +152,8 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): if _hashlib: # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. - self.assertTrue(hasattr(_hashlib, 'openssl_md5')) - self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) + self.assertHasAttr(_hashlib, 'openssl_md5') + self.assertHasAttr(_hashlib, 'openssl_sha1') for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c9b37fcd8f6327..57dc7a24af94b8 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -786,12 +786,12 @@ def test_getfile(self): def test_getfile_builtin_module(self): with self.assertRaises(TypeError) as e: inspect.getfile(sys) - self.assertTrue(str(e.exception).startswith(':5: EncodingWarning: ")) - self.assertTrue( - warnings[1].startswith(b":8: EncodingWarning: ")) + self.assertStartsWith(warnings[0], b":5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b":8: EncodingWarning: ") def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" @@ -4834,7 +4832,7 @@ def on_alarm(*args): os.read(r, len(data) * 100) exc = cm.exception if isinstance(exc, RuntimeError): - self.assertTrue(str(exc).startswith("reentrant call"), str(exc)) + self.assertStartsWith(str(exc), "reentrant call") finally: signal.alarm(0) wio.close() diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 7c1696cc66d12b..79c44af2fbf0e1 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -102,7 +102,7 @@ def test_not_serializable(self): with self.assertRaisesRegex(TypeError, 'Object of type module is not JSON serializable') as cm: self.dumps(sys) - self.assertFalse(hasattr(cm.exception, '__notes__')) + self.assertNotHasAttr(cm.exception, '__notes__') with self.assertRaises(TypeError) as cm: self.dumps([1, [2, 3, sys]]) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 72cde3f0d6c1bd..9ea2679c77ec17 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -160,7 +160,7 @@ def test_help_flag(self): rc, out, err = assert_python_ok('-m', self.module, '-h', PYTHON_COLORS='0') self.assertEqual(rc, 0) - self.assertTrue(out.startswith(b'usage: ')) + self.assertStartsWith(out, b'usage: ') self.assertEqual(err, b'') def test_sort_keys_flag(self): diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 173fc743cf68ae..caa1603c78eb01 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -443,7 +443,7 @@ def test_search_major_3(self): except subprocess.CalledProcessError: raise unittest.SkipTest("requires at least one Python 3.x install") self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") def test_search_major_3_32(self): try: @@ -453,8 +453,8 @@ def test_search_major_3_32(self): raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") raise self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) - self.assertTrue(data["env.tag"].endswith("-32"), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") + self.assertEndsWith(data["env.tag"], "-32") def test_search_major_2(self): try: @@ -463,7 +463,7 @@ def test_search_major_2(self): if not is_installed("2.7"): raise unittest.SkipTest("requires at least one Python 2.x install") self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("2."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "2.") def test_py_default(self): with self.py_ini(TEST_PY_DEFAULTS): diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 9ffb93e797dd80..e93c3c37354e27 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1025,12 +1025,12 @@ def test_peek(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek() self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek(10) self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) def test_peek_bad_args(self): diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 95629ed862d6eb..63998a86c45b53 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -265,8 +265,8 @@ def test_iterator(self): memio = self.ioclass(buf * 10) self.assertEqual(iter(memio), memio) - self.assertTrue(hasattr(memio, '__iter__')) - self.assertTrue(hasattr(memio, '__next__')) + self.assertHasAttr(memio, '__iter__') + self.assertHasAttr(memio, '__next__') i = 0 for line in memio: self.assertEqual(line, buf) diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 9f131a9110dccb..4204a6a47d2a81 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -147,7 +147,7 @@ def test_fromkeys(self): def test_abc(self): OrderedDict = self.OrderedDict self.assertIsInstance(OrderedDict(), MutableMapping) - self.assertTrue(issubclass(OrderedDict, MutableMapping)) + self.assertIsSubclass(OrderedDict, MutableMapping) def test_clear(self): OrderedDict = self.OrderedDict @@ -314,14 +314,14 @@ def check(dup): check(dup) self.assertIs(dup.x, od.x) self.assertIs(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(od) check(dup) self.assertEqual(dup.x, od.x) self.assertIsNot(dup.x, od.x) self.assertEqual(dup.z, od.z) self.assertIsNot(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') # pickle directly pulls the module, so we have to fake it with replaced_module('collections', self.module): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -330,7 +330,7 @@ def check(dup): check(dup) self.assertEqual(dup.x, od.x) self.assertEqual(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') check(eval(repr(od))) update_test = OrderedDict() update_test.update(od) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 333179a71e3cdc..88b5b0e6e358bb 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -818,7 +818,7 @@ def test_15261(self): self.assertEqual(ctx.exception.errno, errno.EBADF) def check_file_attributes(self, result): - self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertHasAttr(result, 'st_file_attributes') self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) @@ -2181,7 +2181,7 @@ def test_getrandom0(self): self.assertEqual(empty, b'') def test_getrandom_random(self): - self.assertTrue(hasattr(os, 'GRND_RANDOM')) + self.assertHasAttr(os, 'GRND_RANDOM') # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare # resource /dev/random @@ -5431,8 +5431,8 @@ def test_fsencode_fsdecode(self): def test_pathlike(self): self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) - self.assertTrue(issubclass(FakePath, os.PathLike)) - self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + self.assertIsSubclass(FakePath, os.PathLike) + self.assertIsInstance(FakePath('x'), os.PathLike) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -5458,8 +5458,8 @@ def test_pathlike_subclasshook(self): # true on abstract implementation. class A(os.PathLike): pass - self.assertFalse(issubclass(FakePath, A)) - self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertNotIsSubclass(FakePath, A) + self.assertIsSubclass(FakePath, os.PathLike) def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) @@ -5469,7 +5469,7 @@ class A(os.PathLike): __slots__ = () def __fspath__(self): return '' - self.assertFalse(hasattr(A(), '__dict__')) + self.assertNotHasAttr(A(), '__dict__') def test_fspath_set_to_None(self): class Foo: diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 37ef9fa1946376..13356b4cfe082b 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -77,8 +77,8 @@ def needs_symlinks(fn): class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + self.assertIsSubclass(pathlib.UnsupportedOperation, NotImplementedError) + self.assertIsInstance(pathlib.UnsupportedOperation(), NotImplementedError) class LazyImportTest(unittest.TestCase): @@ -300,8 +300,8 @@ def test_repr_common(self): clsname = p.__class__.__name__ r = repr(p) # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, clsname + '(') + self.assertEndsWith(r, ')') inner = r[len(clsname) + 1 : -1] self.assertEqual(eval(inner), p.as_posix()) diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 47f51f1979faab..0a9ba578673b39 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -316,7 +316,7 @@ def negzero(): return -(1.0-1.0) for instr in dis.get_instructions(negzero): - self.assertFalse(instr.opname.startswith('UNARY_')) + self.assertNotStartsWith(instr.opname, 'UNARY_') self.check_lnotab(negzero) def test_constant_folding_binop(self): diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1095e7303c188f..aa01a9b8f7ed87 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -387,10 +387,10 @@ def test_with_stmt_with_paren(self) -> None: test_source = """ stmt = "with (\\n a as b,\\n c as d\\n): pass" the_ast = parse.parse_string(stmt, mode=1) - self.assertTrue(ast_dump(the_ast).startswith( + self.assertStartsWith(ast_dump(the_ast), "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" - )) + ) """ self.run_test(grammar_source, test_source) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index d8606521345390..d912c55812397d 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -91,10 +91,8 @@ def test_gather(self) -> None: """ rules = parse_string(grammar, GrammarParser).rules self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") - self.assertTrue( - repr(rules["start"]).startswith( - "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" - ) + self.assertStartsWith(repr(rules["start"]), + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" ) self.assertEqual(str(rules["thing"]), "thing: NUMBER") parser_class = make_parser(grammar) diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index c176e505155b90..21d097dbb559ec 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -93,9 +93,7 @@ def baz(): perf_line, f"Could not find {expected_symbol} in perf file" ) perf_addr = perf_line.split(" ")[0] - self.assertFalse( - perf_addr.startswith("0x"), "Address should not be prefixed with 0x" - ) + self.assertNotStartsWith(perf_addr, "0x") self.assertTrue( set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters", diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 742ca8de1bea8c..e2384b33345a45 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -611,10 +611,10 @@ def test_name_mapping(self): with self.subTest(((module3, name3), (module2, name2))): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, OSError)) + self.assertIsSubclass(attr, OSError) elif (module2, name2) == ('exceptions', 'ImportError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, ImportError)) + self.assertIsSubclass(attr, ImportError) else: module, name = mapping(module2, name2) if module3[:1] != '_': diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 818e807dd3a6fb..719c4feace6544 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -410,7 +410,7 @@ def test_win32_ver(self): for v in version.split('.'): int(v) # should not fail if csd: - self.assertTrue(csd.startswith('SP'), msg=csd) + self.assertStartsWith(csd, 'SP') if ptype: if os.cpu_count() > 1: self.assertIn('Multiprocessor', ptype) diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 0817d0a87a38b1..628920e34b586f 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1107,7 +1107,7 @@ def test_lchmod_dir_symlink(self): def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) - self.assertTrue(hasattr(st, 'st_flags')) + self.assertHasAttr(st, 'st_flags') # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = st.st_flags | stat.UF_IMMUTABLE @@ -1143,7 +1143,7 @@ def test_lchflags_regular_file(self): def test_lchflags_symlink(self): testfn_st = os.stat(os_helper.TESTFN) - self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.assertHasAttr(testfn_st, 'st_flags') self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) @@ -2218,12 +2218,12 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertHasAttr(os, "pwritev") + self.assertHasAttr(os, "preadv") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertNotHasAttr(os, "pwritev") + self.assertNotHasAttr(os, "preadv") def test_stat(self): self._verify_available("HAVE_FSTATAT") diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index cea241b0f200d0..26aefdbf0421dd 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -87,8 +87,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 6dc51e4371d0f6..3c8ed251acaa4d 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -46,7 +46,7 @@ def test_parse_semantics(self): items = pulldom.parseString(SMALL_SAMPLE) evt, node = next(items) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") self.assertEqual(pulldom.START_DOCUMENT, evt) evt, node = next(items) self.assertEqual(pulldom.START_ELEMENT, evt) @@ -192,7 +192,7 @@ def _test_thorough(self, pd, before_root=True): evt, node = next(pd) self.assertEqual(pulldom.START_DOCUMENT, evt) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") if before_root: evt, node = next(pd) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index df05cd07d7e249..3e7b2cd0dc9912 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -103,7 +103,7 @@ def ismethod(oclass, obj, name): for name, value in dict.items(): if name in ignore: continue - self.assertHasAttr(module, name, ignore) + self.assertHasAttr(module, name) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index ac88b3c6f13d5e..281b24eaa36b80 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1380,7 +1380,7 @@ def test_modules_search_builtin(self): helper('modules garbage') result = help_io.getvalue() - self.assertTrue(result.startswith(expected)) + self.assertStartsWith(result, expected) def test_importfile(self): try: diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 43957f525f10c0..bd76d636e4f0fc 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -1415,27 +1415,27 @@ class CommandLineTest(unittest.TestCase): def test_parse_args(self): args, help_text = random._parse_args(shlex.split("--choice a b c")) self.assertEqual(args.choice, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--integer 5")) self.assertEqual(args.integer, 5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--float 2.5")) self.assertEqual(args.float, 2.5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("a b c")) self.assertEqual(args.input, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("5")) self.assertEqual(args.input, ["5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("2.5")) self.assertEqual(args.input, ["2.5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") def test_main(self): for command, expected in [ diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f79a6149078996..e9128ac1d9762d 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2868,11 +2868,11 @@ def test_long_pattern(self): pattern = 'Very %spattern' % ('long ' * 1000) r = repr(re.compile(pattern)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertStartsWith(r, "re.compile('Very long long lon") r = repr(re.compile(pattern, re.I)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") - self.assertEqual(r[-16:], ", re.IGNORECASE)") + self.assertStartsWith(r, "re.compile('Very long long lon") + self.assertEndsWith(r, ", re.IGNORECASE)") def test_flags_repr(self): self.assertEqual(repr(re.I), "re.IGNORECASE") @@ -2951,7 +2951,7 @@ def test_deprecated_modules(self): self.assertEqual(mod.__name__, name) self.assertEqual(mod.__package__, '') for attr in deprecated[name]: - self.assertTrue(hasattr(mod, attr)) + self.assertHasAttr(mod, attr) del sys.modules[name] @cpython_only diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index ffad35092f9916..16623654c29b28 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -173,13 +173,13 @@ def test_instance(self): eq(r(i3), (""%id(i3))) s = r(ClassWithFailingRepr) - self.assertTrue(s.startswith("")) + self.assertStartsWith(s, "") self.assertIn(s.find("..."), [12, 13]) def test_lambda(self): r = repr(lambda x: x) - self.assertTrue(r.startswith("..') # Methods - self.assertTrue(repr(''.split).startswith( - '", "exec") expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ "ordinal not in range(128)" - self.assertTrue(c.exception.args[0].startswith(expected), - msg=c.exception.args[0]) + self.assertStartsWith(c.exception.args[0], expected) def test_file_parse_error_multiline(self): # gh96611: diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 395b2ef88ab622..6a2e1c61f55240 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -539,9 +539,9 @@ def test_openssl_version(self): openssl_ver = f"OpenSSL {major:d}.{minor:d}.{patch:d}" else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" - self.assertTrue( - s.startswith((openssl_ver, libressl_ver, "AWS-LC")), - (s, t, hex(n)) + self.assertStartsWith( + s, (openssl_ver, libressl_ver, "AWS-LC"), + (t, hex(n)) ) @support.cpython_only @@ -1668,7 +1668,7 @@ def test_lib_reason(self): regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue("NO_START_LINE" in s, s) + self.assertIn("NO_START_LINE", s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1683,7 +1683,7 @@ def test_subclass(self): with self.assertRaises(ssl.SSLWantReadError) as cm: c.do_handshake() s = str(cm.exception) - self.assertTrue(s.startswith("The operation did not complete (read)"), s) + self.assertStartsWith(s, "The operation did not complete (read)") # For compatibility self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 49013a4bcd8af6..5fd25d5012c425 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -157,7 +157,7 @@ def test_mode(self): os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() - self.assertEqual(modestr[:3], '-rw') + self.assertStartsWith(modestr, '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) @@ -256,7 +256,7 @@ def test_flags_consistent(self): "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): for key, value in sorted(self.file_attributes.items()): - self.assertTrue(hasattr(self.statmod, key), key) + self.assertHasAttr(self.statmod, key) modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) @@ -314,7 +314,7 @@ def test_macosx_attribute_values(self): self.assertEqual(self.statmod.S_ISGID, 0o002000) self.assertEqual(self.statmod.S_ISVTX, 0o001000) - self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertNotHasAttr(self.statmod, "S_ISTXT") self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index c69baa4bf4d1b1..5980f939185965 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -645,7 +645,7 @@ def do_test(self, args): def test_numerictestcase_is_testcase(self): # Ensure that NumericTestCase actually is a TestCase. - self.assertTrue(issubclass(NumericTestCase, unittest.TestCase)) + self.assertIsSubclass(NumericTestCase, unittest.TestCase) def test_error_msg_numeric(self): # Test the error message generated for numeric comparisons. @@ -683,32 +683,23 @@ class GlobalsTest(unittest.TestCase): def test_meta(self): # Test for the existence of metadata. for meta in self.expected_metadata: - self.assertTrue(hasattr(self.module, meta), - "%s not present" % meta) + self.assertHasAttr(self.module, meta) def test_check_all(self): # Check everything in __all__ exists and is public. module = self.module for name in module.__all__: # No private names in __all__: - self.assertFalse(name.startswith("_"), + self.assertNotStartsWith(name, "_", 'private name "%s" in __all__' % name) # And anything in __all__ must exist: - self.assertTrue(hasattr(module, name), - 'missing name "%s" in __all__' % name) + self.assertHasAttr(module, name) class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): - errmsg = ( - "Expected StatisticsError to be a ValueError, but got a" - " subclass of %r instead." - ) - self.assertTrue(hasattr(statistics, 'StatisticsError')) - self.assertTrue( - issubclass(statistics.StatisticsError, ValueError), - errmsg % statistics.StatisticsError.__base__ - ) + self.assertHasAttr(statistics, 'StatisticsError') + self.assertIsSubclass(statistics.StatisticsError, ValueError) # === Tests for private utility functions === diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index d0bc0bd7b61520..9622151143cd78 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -42,7 +42,7 @@ def test_repr(self): # os.stat() gives a complicated struct sequence. st = os.stat(__file__) rep = repr(st) - self.assertTrue(rep.startswith("os.stat_result")) + self.assertStartsWith(rep, "os.stat_result") self.assertIn("st_mode=", rep) self.assertIn("st_ino=", rep) self.assertIn("st_dev=", rep) @@ -307,7 +307,7 @@ def test_copy_replace_with_invisible_fields(self): self.assertEqual(t5.tm_mon, 2) # named invisible fields - self.assertTrue(hasattr(t, 'tm_zone'), f"{t} has no attribute 'tm_zone'") + self.assertHasAttr(t, 'tm_zone') with self.assertRaisesRegex(AttributeError, 'readonly attribute'): t.tm_zone = 'some other zone' self.assertEqual(t2.tm_zone, t.tm_zone) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ca35804fb36076..f0e350c71f60ea 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -1178,7 +1178,7 @@ def test_universal_newlines_communicate_stdin_stdout_stderr(self): self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) # Python debug build push something like "[42442 refs]\n" # to stderr at exit of subprocess. - self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + self.assertStartsWith(stderr, "eline2\neline6\neline7\n") def test_universal_newlines_communicate_encodings(self): # Check that universal newlines mode works for various encodings, @@ -1510,7 +1510,7 @@ def test_issue8780(self): "[sys.executable, '-c', 'print(\"Hello World!\")'])", 'assert retcode == 0')) output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + self.assertStartsWith(output, b'Hello World!') def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the @@ -1835,8 +1835,8 @@ def test_encoding_warning(self): capture_output=True) lines = cp.stderr.splitlines() self.assertEqual(len(lines), 2, lines) - self.assertTrue(lines[0].startswith(b":2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b":3: EncodingWarning: ")) + self.assertStartsWith(lines[0], b":2: EncodingWarning: ") + self.assertStartsWith(lines[1], b":3: EncodingWarning: ") def _get_test_grp_name(): diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 5cef612a340be9..193c8b7d7f3e13 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -547,11 +547,11 @@ def test_special_methods(self): self.assertEqual(s.__reduce__, e.__reduce__) self.assertEqual(s.__reduce_ex__, e.__reduce_ex__) self.assertEqual(s.__getstate__, e.__getstate__) - self.assertFalse(hasattr(s, '__getnewargs__')) - self.assertFalse(hasattr(s, '__getnewargs_ex__')) - self.assertFalse(hasattr(s, '__setstate__')) - self.assertFalse(hasattr(s, '__copy__')) - self.assertFalse(hasattr(s, '__deepcopy__')) + self.assertNotHasAttr(s, '__getnewargs__') + self.assertNotHasAttr(s, '__getnewargs_ex__') + self.assertNotHasAttr(s, '__setstate__') + self.assertNotHasAttr(s, '__copy__') + self.assertNotHasAttr(s, '__deepcopy__') def test_pickling(self): e = E() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8446da03e3645b..e48a2464ee5977 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -407,10 +407,10 @@ class Obj: with support.swap_attr(obj, "y", 5) as y: self.assertEqual(obj.y, 5) self.assertIsNone(y) - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') with support.swap_attr(obj, "y", 5): del obj.y - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') def test_swap_item(self): D = {"x":1} diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index fb1c8492a64d38..795d1ecbb59f8f 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -57,7 +57,7 @@ def test_original_displayhook(self): dh(None) self.assertEqual(out.getvalue(), "") - self.assertTrue(not hasattr(builtins, "_")) + self.assertNotHasAttr(builtins, "_") # sys.displayhook() requires arguments self.assertRaises(TypeError, dh) @@ -172,7 +172,7 @@ def test_original_excepthook(self): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertEndsWith(err.getvalue(), "ValueError: 42\n") self.assertRaises(TypeError, sys.__excepthook__) @@ -192,7 +192,7 @@ def test_excepthook_bytes_filename(self): err = err.getvalue() self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) self.assertIn(""" text\n""", err) - self.assertTrue(err.endswith("SyntaxError: msg\n")) + self.assertEndsWith(err, "SyntaxError: msg\n") def test_excepthook(self): with test.support.captured_output("stderr") as stderr: @@ -269,8 +269,7 @@ def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) self.assertEqual(rc, 1) self.assertEqual(out, b'') - self.assertTrue(err.startswith(expected), - "%s doesn't start with %s" % (ascii(err), ascii(expected))) + self.assertStartsWith(err, expected) # test that stderr buffer is flushed before the exit message is written # into stderr @@ -437,7 +436,7 @@ def test_call_tracing(self): @unittest.skipUnless(hasattr(sys, "setdlopenflags"), 'test needs sys.setdlopenflags()') def test_dlopenflags(self): - self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertHasAttr(sys, "getdlopenflags") self.assertRaises(TypeError, sys.getdlopenflags, 42) oldflags = sys.getdlopenflags() self.assertRaises(TypeError, sys.setdlopenflags) @@ -623,8 +622,7 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue((sourceline.startswith("if leave_g.wait(") or - sourceline.startswith("g_raised.set()"))) + self.assertStartsWith(sourceline, ("if leave_g.wait(", "g_raised.set()")) finally: # Reap the spawned thread. leave_g.set() @@ -860,7 +858,7 @@ def test_sys_flags(self): "hash_randomization", "isolated", "dev_mode", "utf8_mode", "warn_default_encoding", "safe_path", "int_max_str_digits") for attr in attrs: - self.assertTrue(hasattr(sys.flags, attr), attr) + self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) @@ -1072,10 +1070,10 @@ def test_implementation(self): levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} - self.assertTrue(hasattr(sys.implementation, 'name')) - self.assertTrue(hasattr(sys.implementation, 'version')) - self.assertTrue(hasattr(sys.implementation, 'hexversion')) - self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + self.assertHasAttr(sys.implementation, 'name') + self.assertHasAttr(sys.implementation, 'version') + self.assertHasAttr(sys.implementation, 'hexversion') + self.assertHasAttr(sys.implementation, 'cache_tag') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -1419,7 +1417,7 @@ def __del__(self): else: self.assertIn("ValueError", report) self.assertIn("del is broken", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") def test_original_unraisablehook_exception_qualname(self): # See bpo-41031, bpo-45083. diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 963cf753ce6178..d30f69ded6643a 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -186,7 +186,7 @@ def test_posix_venv_scheme(self): # The include directory on POSIX isn't exactly the same as before, # but it is "within" sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + self.assertStartsWith(sysconfig_includedir, incpath + os.sep) def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module @@ -569,8 +569,7 @@ def test_linux_ext_suffix(self): expected_suffixes = 'i386-linux-gnu.so', 'x86_64-linux-gnux32.so', 'i386-linux-musl.so' else: # 8 byte pointer size expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' - self.assertTrue(suffix.endswith(expected_suffixes), - f'unexpected suffix {suffix!r}') + self.assertEndsWith(suffix, expected_suffixes) @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') def test_android_ext_suffix(self): @@ -582,13 +581,12 @@ def test_android_ext_suffix(self): "aarch64": "aarch64-linux-android", "armv7l": "arm-linux-androideabi", }[machine] - self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), - f"{machine=}, {suffix=}") + self.assertEndsWith(suffix, f"-{expected_triplet}.so") @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(suffix.endswith('-darwin.so'), suffix) + self.assertEndsWith(suffix, '-darwin.so') def test_always_set_py_debug(self): self.assertIn('Py_DEBUG', sysconfig.get_config_vars()) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2018a20afd1b18..cf218a2bf14369 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -1650,7 +1650,7 @@ def test_cwd(self): try: for t in tar: if t.name != ".": - self.assertTrue(t.name.startswith("./"), t.name) + self.assertStartsWith(t.name, "./") finally: tar.close() diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index d46d3c0f040601..52b13b98cbcce5 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -516,11 +516,11 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): (fd1, name1) = self.make_temp() os.close(fd1) - self.assertTrue(name1.endswith('aaa')) + self.assertEndsWith(name1, 'aaa') (fd2, name2) = self.make_temp() os.close(fd2) - self.assertTrue(name2.endswith('bbb')) + self.assertEndsWith(name2, 'bbb') def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with @@ -528,11 +528,11 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('aaa')) + self.assertEndsWith(dir, 'aaa') (fd, name) = self.make_temp() os.close(fd) - self.assertTrue(name.endswith('bbb')) + self.assertEndsWith(name, 'bbb') class TestGetTempPrefix(BaseTestCase): @@ -828,9 +828,9 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): file = tempfile.NamedTemporaryFile(delete=False) file.close() - self.assertTrue(file.name.endswith('aaa')) + self.assertEndsWith(file.name, 'aaa') dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('bbb')) + self.assertEndsWith(dir, 'bbb') def test_collision_with_existing_directory(self): # mkdtemp tries another name when a directory with @@ -838,9 +838,9 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir1 = tempfile.mkdtemp() - self.assertTrue(dir1.endswith('aaa')) + self.assertEndsWith(dir1, 'aaa') dir2 = tempfile.mkdtemp() - self.assertTrue(dir2.endswith('bbb')) + self.assertEndsWith(dir2, 'bbb') def test_for_tempdir_is_bytes_issue40701_api_warts(self): orig_tempdir = tempfile.tempdir diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index e5d11cf84d2a66..ce8392a6ccdbd6 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -290,8 +290,8 @@ def test_ioctl_constants(self): self.assertGreaterEqual(value, 0) def test_exception(self): - self.assertTrue(issubclass(termios.error, Exception)) - self.assertFalse(issubclass(termios.error, OSError)) + self.assertIsSubclass(termios.error, Exception) + self.assertNotIsSubclass(termios.error, OSError) if __name__ == '__main__': diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d06f65270efe79..5312faa50774ec 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -761,17 +761,17 @@ def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. lt = time.localtime() - self.assertTrue(hasattr(lt, "tm_gmtoff")) - self.assertTrue(hasattr(lt, "tm_zone")) + self.assertHasAttr(lt, "tm_gmtoff") + self.assertHasAttr(lt, "tm_zone") # See if the offset and zone are similar to the module # attributes. if lt.tm_gmtoff is None: - self.assertTrue(not hasattr(time, "timezone")) + self.assertNotHasAttr(time, "timezone") else: self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) if lt.tm_zone is None: - self.assertTrue(not hasattr(time, "tzname")) + self.assertNotHasAttr(time, "tzname") else: self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) @@ -1184,11 +1184,11 @@ def test_clock_functions(self): if mac_ver >= (10, 12): for name in clock_names: - self.assertTrue(hasattr(time, name), f"time.{name} is not available") + self.assertHasAttr(time, name) else: for name in clock_names: - self.assertFalse(hasattr(time, name), f"time.{name} is available") + self.assertNotHasAttr(time, name) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index f5ae0a84eb3506..2aeebea9f93d43 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -222,8 +222,8 @@ def test_repeat_function_zero_iters(self): def assert_exc_string(self, exc_string, expected_exc_name): exc_lines = exc_string.splitlines() self.assertGreater(len(exc_lines), 2) - self.assertTrue(exc_lines[0].startswith('Traceback')) - self.assertTrue(exc_lines[-1].startswith(expected_exc_name)) + self.assertStartsWith(exc_lines[0], 'Traceback') + self.assertStartsWith(exc_lines[-1], expected_exc_name) def test_print_exc(self): s = io.StringIO() diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index ebb9e00ff91bf0..46b01e6f131290 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -58,7 +58,7 @@ def _test_widget(self, constructor): destroy_default_root() tkinter.NoDefaultRoot() self.assertRaises(RuntimeError, constructor) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') def destroy_default_root(): diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 96ea3f0117ca03..0c76e07066f8a8 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -497,7 +497,7 @@ def test_info_patchlevel(self): self.assertEqual(vi.serial, 0) else: self.assertEqual(vi.micro, 0) - self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + self.assertStartsWith(str(vi), f'{vi.major}.{vi.minor}') def test_embedded_null(self): widget = tkinter.Entry(self.root) @@ -609,7 +609,7 @@ def test_focus(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -642,7 +642,7 @@ def test_configure(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -676,7 +676,7 @@ def test_event_generate_key_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertIsInstance(e.state, int) self.assertNotEqual(e.state, 0) @@ -747,7 +747,7 @@ def test_event_generate_button_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, 1) self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -781,7 +781,7 @@ def test_event_generate_motion(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0x100) self.assertEqual(e.char, '??') @@ -814,7 +814,7 @@ def test_event_generate_mouse_wheel(self): self.assertIs(e.widget, f) self.assertIsInstance(e.serial, int) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.time, 0) self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) @@ -849,7 +849,7 @@ def test_generate_event_virtual_event(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -1308,17 +1308,17 @@ def test_no_default_root(self): self.assertIs(tkinter._default_root, root) tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') # repeated call is no-op tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root = tkinter.Tk() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() def test_getboolean(self): diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index ee8939f62d082c..c40c45594f4d80 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -344,7 +344,7 @@ def test_longargs(self): todo = set(t.name[1:]) self.assertEqual(len(t.args.args) + len(t.args.posonlyargs), len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) - self.assertTrue(t.name.startswith('f'), t.name) + self.assertStartsWith(t.name, 'f') for index, c in enumerate(t.name[1:]): todo.remove(c) if c == 'v': diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3552b6b4ef846c..3097c7ddf05901 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -827,15 +827,15 @@ def test_instancecheck_and_subclasscheck(self): self.assertIsInstance(True, x) self.assertIsInstance('a', x) self.assertNotIsInstance(None, x) - self.assertTrue(issubclass(int, x)) - self.assertTrue(issubclass(bool, x)) - self.assertTrue(issubclass(str, x)) - self.assertFalse(issubclass(type(None), x)) + self.assertIsSubclass(int, x) + self.assertIsSubclass(bool, x) + self.assertIsSubclass(str, x) + self.assertNotIsSubclass(type(None), x) for x in (int | None, typing.Union[int, None]): with self.subTest(x=x): self.assertIsInstance(None, x) - self.assertTrue(issubclass(type(None), x)) + self.assertIsSubclass(type(None), x) for x in ( int | collections.abc.Mapping, @@ -844,8 +844,8 @@ def test_instancecheck_and_subclasscheck(self): with self.subTest(x=x): self.assertIsInstance({}, x) self.assertNotIsInstance((), x) - self.assertTrue(issubclass(dict, x)) - self.assertFalse(issubclass(list, x)) + self.assertIsSubclass(dict, x) + self.assertNotIsSubclass(list, x) def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -857,7 +857,7 @@ def test_instancecheck_and_subclasscheck_order(self): for x in will_resolve: with self.subTest(x=x): self.assertIsInstance(1, x) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) wont_resolve = ( T | int, @@ -890,7 +890,7 @@ class BadMeta(type): def __subclasscheck__(cls, sub): 1/0 x = int | BadMeta('A', (), {}) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) self.assertRaises(ZeroDivisionError, issubclass, list, x) def test_or_type_operator_with_TypeVar(self): @@ -1399,7 +1399,7 @@ def test_new_class_basics(self): def test_new_class_subclass(self): C = types.new_class("C", (int,)) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) def test_new_class_meta(self): Meta = self.Meta @@ -1444,7 +1444,7 @@ def func(ns): bases=(int,), kwds=dict(metaclass=Meta, z=2), exec_body=func) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) self.assertIsInstance(C, Meta) self.assertEqual(C.x, 0) self.assertEqual(C.y, 1) diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index a04af55f3fc0ae..d66cab146af246 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1989,7 +1989,7 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) - def testAssertStartswith(self): + def testAssertStartsWith(self): self.assertStartsWith('ababahalamaha', 'ababa') self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertStartsWith(UserString('ababahalamaha'), 'ababa') @@ -2034,7 +2034,7 @@ def testAssertStartswith(self): self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotStartswith(self): + def testAssertNotStartsWith(self): self.assertNotStartsWith('ababahalamaha', 'amaha') self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') @@ -2079,7 +2079,7 @@ def testAssertNotStartswith(self): self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertEndswith(self): + def testAssertEndsWith(self): self.assertEndsWith('ababahalamaha', 'amaha') self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertEndsWith(UserString('ababahalamaha'), 'amaha') @@ -2124,7 +2124,7 @@ def testAssertEndswith(self): self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotEndswith(self): + def testAssertNotEndsWith(self): self.assertNotEndsWith('ababahalamaha', 'ababa') self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index ace84ef564df72..75de9ea252de98 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -166,7 +166,7 @@ def test_update(self): def test_missing(self): # Make sure UserDict doesn't have a __missing__ method - self.assertEqual(hasattr(collections.UserDict, "__missing__"), False) + self.assertNotHasAttr(collections.UserDict, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index adc86a49b0668d..12c30e178aeb51 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -774,7 +774,7 @@ def test_activate_shell_script_has_no_dos_newlines(self): with open(script_path, 'rb') as script: for i, line in enumerate(script, 1): error_message = f"CR LF found in line {i}" - self.assertFalse(line.endswith(b'\r\n'), error_message) + self.assertNotEndsWith(line, b'\r\n', error_message) @requireVenvCreate def test_scm_ignore_files_git(self): @@ -978,7 +978,7 @@ def do_test_with_pip(self, system_site_packages): self.assertEqual(err, "") out = out.decode("latin-1") # Force to text, prevent decoding errors expected_version = "pip {}".format(ensurepip.version()) - self.assertEqual(out[:len(expected_version)], expected_version) + self.assertStartsWith(out, expected_version) env_dir = os.fsencode(self.env_dir).decode("latin-1") self.assertIn(env_dir, out) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 05710c469348c4..5c3b1250ceb045 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -102,7 +102,7 @@ class PublicAPITests(BaseTest): """ def test_module_all_attribute(self): - self.assertTrue(hasattr(self.module, '__all__')) + self.assertHasAttr(self.module, '__all__') target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings", "deprecated"] @@ -735,7 +735,7 @@ class CWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_accelerated(self): self.assertIsNot(original_warnings, self.module) - self.assertFalse(hasattr(self.module.warn, '__code__')) + self.assertNotHasAttr(self.module.warn, '__code__') class PyWarnTests(WarnTests, unittest.TestCase): module = py_warnings @@ -744,7 +744,7 @@ class PyWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_pure_python(self): self.assertIsNot(original_warnings, self.module) - self.assertTrue(hasattr(self.module.warn, '__code__')) + self.assertHasAttr(self.module.warn, '__code__') class WCmdLineTests(BaseTest): @@ -1528,12 +1528,12 @@ def test_late_resource_warning(self): # (_warnings will try to import it) code = "f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) # import the warnings module code = "import warnings; f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) class AsyncTests(BaseTest): diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4faad6629fe23c..4c7c900eb56ae1 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -432,7 +432,7 @@ def check_proxy(self, o, proxy): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -442,7 +442,7 @@ def check_proxy(self, o, proxy): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -1108,7 +1108,7 @@ def meth(self): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 76e8e5c8ab7d3c..c1e4f9c8366e58 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -466,7 +466,7 @@ def test_copying(self): self.assertIsNot(dup, s) self.assertIs(dup.x, s.x) self.assertIs(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(s) self.assertIsInstance(dup, cls) @@ -476,7 +476,7 @@ def test_copying(self): self.assertIsNot(dup.x, s.x) self.assertEqual(dup.z, s.z) self.assertIsNot(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') if __name__ == "__main__": diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index d9076e77c158a2..1bae884ed9ae3e 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -17,9 +17,9 @@ class WindowsConsoleIOTests(unittest.TestCase): def test_abc(self): - self.assertTrue(issubclass(ConIO, io.RawIOBase)) - self.assertFalse(issubclass(ConIO, io.BufferedIOBase)) - self.assertFalse(issubclass(ConIO, io.TextIOBase)) + self.assertIsSubclass(ConIO, io.RawIOBase) + self.assertNotIsSubclass(ConIO, io.BufferedIOBase) + self.assertNotIsSubclass(ConIO, io.TextIOBase) def test_open_fd(self): self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index fd7abd1782ec4d..f16611b29a2658 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -679,7 +679,7 @@ def testSingleComplexTarget(self): class C: pass blah = C() with mock_contextmanager_generator() as blah.foo: - self.assertEqual(hasattr(blah, "foo"), True) + self.assertHasAttr(blah, "foo") def testMultipleComplexTargets(self): class C: diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index ac7c9cb3a5a493..90eb40439d4b4a 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -70,8 +70,8 @@ def test_wmi_query_overflow(self): def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") - self.assertFalse(r.startswith("\0"), r) - self.assertFalse(r.endswith("\0"), r) + self.assertNotStartsWith(r, "\0") + self.assertNotEndsWith(r, "\0") it = iter(r.split("\0")) try: while True: diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index b047f7b06f85d3..e04a4d2c2218a3 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -149,9 +149,9 @@ def bad_app(environ,start_response): start_response("200 OK", ('Content-Type','text/plain')) return ["Hello, world!"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError: Headers (('Content-Type', 'text/plain')) must" @@ -174,9 +174,9 @@ def bad_app(environ, start_response): for status, exc_message in tests: with self.subTest(status=status): out, err = run_amock(create_bad_app(status)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual(err.splitlines()[-2], exc_message) def test_wsgi_input(self): @@ -185,9 +185,9 @@ def bad_app(e,s): s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) return [b"data"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError" ) @@ -200,7 +200,7 @@ def app(e, s): ]) return [b"data"] out, err = run_amock(validator(app)) - self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + self.assertEndsWith(err, '"GET / HTTP/1.0" 200 4\n') ver = sys.version.split()[0].encode('ascii') py = python_implementation().encode('ascii') pyver = py + b"/" + ver diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 8f2779520070d2..38be2cd437f200 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -225,8 +225,7 @@ def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") direlem = dir(element) for attr in 'tag', 'attrib', 'text', 'tail': - self.assertTrue(hasattr(element, attr), - msg='no %s member' % attr) + self.assertHasAttr(element, attr) self.assertIn(attr, direlem, msg='no %s visible by dir' % attr) @@ -251,7 +250,7 @@ def check_element(element): # Make sure all standard element methods exist. def check_method(method): - self.assertTrue(hasattr(method, '__call__'), + self.assertHasAttr(method, '__call__', msg="%s not callable" % method) check_method(element.append) diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index 6dbfb3f439393c..b52e78bc4fb7e0 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -31,7 +31,7 @@ def test_foo(self): self.assertEqual(self.module.foo(1, 2), 3) def test_str(self): - self.assertTrue(issubclass(self.module.Str, str)) + self.assertIsSubclass(self.module.Str, str) self.assertIsNot(self.module.Str, str) custom_string = self.module.Str("abcd") diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index d4766c59a102db..8fb0a68deba535 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -259,7 +259,7 @@ def test_pack_to_fileobj(self): (source / '__main__.py').touch() target = io.BytesIO() zipapp.create_archive(str(source), target, interpreter='python') - self.assertTrue(target.getvalue().startswith(b'#!python\n')) + self.assertStartsWith(target.getvalue(), b'#!python\n') def test_read_shebang(self): # Test that we can read the shebang line correctly. @@ -300,7 +300,7 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(source), str(target), interpreter='python') new_target = io.BytesIO() zipapp.create_archive(str(target), new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_read_from_pathlike_obj(self): # Test that we can copy an archive using a path-like object @@ -326,7 +326,7 @@ def test_read_from_fileobj(self): new_target = io.BytesIO() temp_archive.seek(0) zipapp.create_archive(temp_archive, new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_remove_shebang(self): # Test that we can remove the shebang from a file. diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index e93603998f979e..ada96813709aea 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3198,7 +3198,7 @@ def test_write_dir(self): with zipfile.ZipFile(TESTFN, "w") as zipf: zipf.write(dirpath) zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zipf.write(dirpath, "y") zinfo = zipf.filelist[1] @@ -3206,7 +3206,7 @@ def test_write_dir(self): self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zinfo = zipf.filelist[1] self.assertTrue(zinfo.filename, "y/") @@ -3226,7 +3226,7 @@ def test_writestr_dir(self): self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("x/")) + self.assertEndsWith(zinfo.filename, "x/") self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) target = os.path.join(TESTFN2, "target") os.mkdir(target) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 1f288c8b45d589..b5b4acf5f850be 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -835,11 +835,11 @@ def doTraceback(self, module): s = io.StringIO() print_tb(tb, 1, s) - self.assertTrue(s.getvalue().endswith( + self.assertEndsWith(s.getvalue(), ' def do_raise(): raise TypeError\n' '' if support.has_no_debug_ranges() else ' ^^^^^^^^^^^^^^^\n' - )) + ) else: raise AssertionError("This ought to be impossible") diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index d2845495c7f8b6..f313e394f49e9b 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -1915,8 +1915,8 @@ class ExtensionBuiltTest(unittest.TestCase): def test_cache_location(self): # The pure Python version stores caches on attributes, but the C # extension stores them in C globals (at least for now) - self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) - self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + self.assertNotHasAttr(c_zoneinfo.ZoneInfo, "_weak_cache") + self.assertHasAttr(py_zoneinfo.ZoneInfo, "_weak_cache") def test_gc_tracked(self): import gc From 59cb829eb9b9f7bb301cc795c9ee8f5e0c7269f2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 14:26:26 +0200 Subject: [PATCH 198/713] [3.14] gh-134455: Fix `build-details.json` to use the ``c_api.headers`` key (GH-134456) (#134504) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Michał Górny Co-authored-by: Filipe Laíns 🇵🇸 --- .../next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst | 2 ++ Tools/build/generate-build-details.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst diff --git a/Misc/NEWS.d/next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst b/Misc/NEWS.d/next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst new file mode 100644 index 00000000000000..08833b3344f20b --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst @@ -0,0 +1,2 @@ +Fixed ``build-details.json`` generation to use the correct ``c_api.headers`` +as defined in :pep:`739`, instead of ``c_api.include``. diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 5dc100b8b05e5e..87e262065ec87b 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -123,7 +123,7 @@ def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: if has_static_library: data['libpython']['static'] = os.path.join(LIBDIR, LIBRARY) - data['c_api']['include'] = INCLUDEDIR + data['c_api']['headers'] = INCLUDEDIR if LIBPC: data['c_api']['pkgconfig_path'] = LIBPC From 85c8c0a003dca9ac48b4e5c71f798085822f8760 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 15:21:05 +0200 Subject: [PATCH 199/713] [3.14] gh-132775: Use _PyObject_GetXIData (With Fallback) (gh-134507) This change includes some semi-related refactoring of queues and channels. (cherry picked from commit d0eedfa10e, gh-134440) Co-authored-by: Eric Snow --- Lib/concurrent/futures/interpreter.py | 55 ++---- Lib/test/support/interpreters/channels.py | 85 +++++---- Lib/test/support/interpreters/queues.py | 144 +++++++-------- Lib/test/test__interpchannels.py | 6 +- Lib/test/test_interpreters/test_channels.py | 40 ++--- Lib/test/test_interpreters/test_queues.py | 188 +++++++------------- Modules/_interpchannelsmodule.c | 176 +++++++++++------- Modules/_interpqueuesmodule.c | 175 ++++++++++-------- Modules/_interpreters_common.h | 54 ++++++ Python/crossinterp.c | 1 + 10 files changed, 486 insertions(+), 438 deletions(-) diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index d17688dc9d7346..a2c4fbfd3fb831 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -36,9 +36,6 @@ def __str__(self): """.strip()) -UNBOUND = 2 # error; this should not happen. - - class WorkerContext(_thread.WorkerContext): @classmethod @@ -47,23 +44,13 @@ def resolve_task(fn, args, kwargs): if isinstance(fn, str): # XXX Circle back to this later. raise TypeError('scripts not supported') - if args or kwargs: - raise ValueError(f'a script does not take args or kwargs, got {args!r} and {kwargs!r}') - data = textwrap.dedent(fn) - kind = 'script' - # Make sure the script compiles. - # Ideally we wouldn't throw away the resulting code - # object. However, there isn't much to be done until - # code objects are shareable and/or we do a better job - # of supporting code objects in _interpreters.exec(). - compile(data, '', 'exec') else: # Functions defined in the __main__ module can't be pickled, # so they can't be used here. In the future, we could possibly # borrow from multiprocessing to work around this. - data = pickle.dumps((fn, args, kwargs)) - kind = 'function' - return (data, kind) + task = (fn, args, kwargs) + data = pickle.dumps(task) + return data if initializer is not None: try: @@ -86,24 +73,20 @@ def _capture_exc(cls, resultsid): except BaseException as exc: # Send the captured exception out on the results queue, # but still leave it unhandled for the interpreter to handle. - err = pickle.dumps(exc) - _interpqueues.put(resultsid, (None, err), 1, UNBOUND) + _interpqueues.put(resultsid, (None, exc)) raise # re-raise @classmethod def _send_script_result(cls, resultsid): - _interpqueues.put(resultsid, (None, None), 0, UNBOUND) + _interpqueues.put(resultsid, (None, None)) @classmethod def _call(cls, func, args, kwargs, resultsid): with cls._capture_exc(resultsid): res = func(*args or (), **kwargs or {}) # Send the result back. - try: - _interpqueues.put(resultsid, (res, None), 0, UNBOUND) - except _interpreters.NotShareableError: - res = pickle.dumps(res) - _interpqueues.put(resultsid, (res, None), 1, UNBOUND) + with cls._capture_exc(resultsid): + _interpqueues.put(resultsid, (res, None)) @classmethod def _call_pickled(cls, pickled, resultsid): @@ -134,8 +117,7 @@ def initialize(self): _interpreters.incref(self.interpid) maxsize = 0 - fmt = 0 - self.resultsid = _interpqueues.create(maxsize, fmt, UNBOUND) + self.resultsid = _interpqueues.create(maxsize) self._exec(f'from {__name__} import WorkerContext') @@ -166,17 +148,8 @@ def finalize(self): pass def run(self, task): - data, kind = task - if kind == 'script': - raise NotImplementedError('script kind disabled') - script = f""" -with WorkerContext._capture_exc({self.resultsid}): -{textwrap.indent(data, ' ')} -WorkerContext._send_script_result({self.resultsid})""" - elif kind == 'function': - script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' - else: - raise NotImplementedError(kind) + data = task + script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' try: self._exec(script) @@ -199,15 +172,13 @@ def run(self, task): continue else: break - (res, excdata), pickled, unboundop = obj + (res, exc), unboundop = obj assert unboundop is None, unboundop - if excdata is not None: + if exc is not None: assert res is None, res - assert pickled assert exc_wrapper is not None - exc = pickle.loads(excdata) raise exc from exc_wrapper - return pickle.loads(res) if pickled else res + return res class BrokenInterpreterPool(_thread.BrokenThreadPool): diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index d2bd93d77f7169..7a2bd7d63f808f 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -55,15 +55,23 @@ def create(*, unbounditems=UNBOUND): """ unbound = _serialize_unbound(unbounditems) unboundop, = unbound - cid = _channels.create(unboundop) - recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) + cid = _channels.create(unboundop, -1) + recv, send = RecvChannel(cid), SendChannel(cid) + send._set_unbound(unboundop, unbounditems) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) - for cid, unbound in _channels.list_all()] + channels = [] + for cid, unboundop, _ in _channels.list_all(): + chan = _, send = RecvChannel(cid), SendChannel(cid) + if not hasattr(send, '_unboundop'): + send._set_unbound(unboundop) + else: + assert send._unbound[0] == op + channels.append(chan) + return channels class _ChannelEnd: @@ -175,16 +183,33 @@ class SendChannel(_ChannelEnd): _end = 'send' - def __new__(cls, cid, *, _unbound=None): - if _unbound is None: - try: - op = _channels.get_channel_defaults(cid) - _unbound = (op,) - except ChannelNotFoundError: - _unbound = _serialize_unbound(UNBOUND) - self = super().__new__(cls, cid) - self._unbound = _unbound - return self +# def __new__(cls, cid, *, _unbound=None): +# if _unbound is None: +# try: +# op = _channels.get_channel_defaults(cid) +# _unbound = (op,) +# except ChannelNotFoundError: +# _unbound = _serialize_unbound(UNBOUND) +# self = super().__new__(cls, cid) +# self._unbound = _unbound +# return self + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _channels.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items @property def is_closed(self): @@ -192,61 +217,61 @@ def is_closed(self): return info.closed or info.closing def send(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. return _channels.send(self._id, obj, unboundop, blocking=False) def send_buffer(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send_buffer(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_buffer_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py index deb8e8613af731..d6a3197d9e0e26 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/test/support/interpreters/queues.py @@ -63,29 +63,34 @@ def _resolve_unbound(flag): return resolved -def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): +def create(maxsize=0, *, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. - "syncobj" sets the default for Queue.put() - and Queue.put_nowait(). - - "unbounditems" likewise sets the default. See Queue.put() for + "unbounditems" sets the default for Queue.put(); see that method for supported values. The default value is UNBOUND, which replaces the unbound item. """ - fmt = _SHARED_ONLY if syncobj else _PICKLED unbound = _serialize_unbound(unbounditems) unboundop, = unbound - qid = _queues.create(maxsize, fmt, unboundop) - return Queue(qid, _fmt=fmt, _unbound=unbound) + qid = _queues.create(maxsize, unboundop, -1) + self = Queue(qid) + self._set_unbound(unboundop, unbounditems) + return self def list_all(): """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) - for qid, fmt, unboundop in _queues.list_all()] + queues = [] + for qid, unboundop, _ in _queues.list_all(): + self = Queue(qid) + if not hasattr(self, '_unbound'): + self._set_unbound(unboundop) + else: + assert self._unbound[0] == unboundop + queues.append(self) + return queues _known_queues = weakref.WeakValueDictionary() @@ -93,28 +98,17 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /, *, _fmt=None, _unbound=None): + def __new__(cls, id, /): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') - if _fmt is None: - if _unbound is None: - _fmt, op = _queues.get_queue_defaults(id) - _unbound = (op,) - else: - _fmt, _ = _queues.get_queue_defaults(id) - elif _unbound is None: - _, op = _queues.get_queue_defaults(id) - _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id - self._fmt = _fmt - self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -143,10 +137,27 @@ def __getnewargs__(self): def __getstate__(self): return None + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + @property def id(self): return self._id + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _queues.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items + @property def maxsize(self): try: @@ -165,77 +176,56 @@ def qsize(self): return _queues.get_count(self._id) def put(self, obj, timeout=None, *, - syncobj=None, - unbound=None, + unbounditems=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. This blocks while the queue is full. - If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue(). - - If "syncobj" is false then all objects are supported, - at the expense of worse performance. - - If "syncobj" is true then the object must be "shareable". - Examples of "shareable" objects include the builtin singletons, - str, and memoryview. One benefit is that such objects are - passed through the queue efficiently. - - The key difference, though, is conceptual: the corresponding - object returned from Queue.get() will be strictly equivalent - to the given obj. In other words, the two objects will be - effectively indistinguishable from each other, even if the - object is mutable. The received object may actually be the - same object, or a copy (immutable values only), or a proxy. - Regardless, the received object should be treated as though - the original has been shared directly, whether or not it - actually is. That's a slightly different and stronger promise - than just (initial) equality, which is all "syncobj=False" - can promise. - - "unbound" controls the behavior of Queue.get() for the given + For most objects, the object received through Queue.get() will + be a new one, equivalent to the original and not sharing any + actual underlying data. The notable exceptions include + cross-interpreter types (like Queue) and memoryview, where the + underlying data is actually shared. Furthermore, some types + can be sent through a queue more efficiently than others. This + group includes various immutable types like int, str, bytes, and + tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable(). + + "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later destroyed. - If "unbound" is None (the default) then it uses the + If "unbounditems" is None (the default) then it uses the queue's default, set with create_queue(), which is usually UNBOUND. - If "unbound" is UNBOUND_ERROR then get() will raise an + If "unbounditems" is UNBOUND_ERROR then get() will raise an ItemInterpreterDestroyed exception if the original interpreter has been destroyed. This does not otherwise affect the queue; the next call to put() will work like normal, returning the next item in the queue. - If "unbound" is UNBOUND_REMOVE then the item will be removed + If "unbounditems" is UNBOUND_REMOVE then the item will be removed from the queue as soon as the original interpreter is destroyed. Be aware that this will introduce an imbalance between put() and get() calls. - If "unbound" is UNBOUND then it is returned by get() in place + If "unbounditems" is UNBOUND then it is returned by get() in place of the unbound item. """ - if syncobj is None: - fmt = self._fmt - else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - if fmt is _PICKLED: - obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj, fmt, unboundop) + _queues.put(self._id, obj, unboundop) except QueueFull as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -243,18 +233,12 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj, *, syncobj=None, unbound=None): - if syncobj is None: - fmt = self._fmt + def put_nowait(self, obj, *, unbounditems=None): + if unbounditems is None: + unboundop = -1 else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) - if fmt is _PICKLED: - obj = pickle.dumps(obj) - _queues.put(self._id, obj, fmt, unboundop) + unboundop, = _serialize_unbound(unbounditems) + _queues.put(self._id, obj, unboundop) def get(self, timeout=None, *, _delay=10 / 1000, # 10 milliseconds @@ -265,7 +249,7 @@ def get(self, timeout=None, *, If the next item's original interpreter has been destroyed then the "next object" is determined by the value of the - "unbound" argument to put(). + "unbounditems" argument to put(). """ if timeout is not None: timeout = int(timeout) @@ -274,7 +258,7 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -284,10 +268,6 @@ def get(self, timeout=None, *, if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj def get_nowait(self): @@ -297,16 +277,12 @@ def get_nowait(self): is the same as get(). """ try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index e4c1ad854514ed..88eee03a3de93a 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -247,7 +247,7 @@ def _run_action(cid, action, end, state): def clean_up_channels(): - for cid, _ in _channels.list_all(): + for cid, _, _ in _channels.list_all(): try: _channels.destroy(cid) except _channels.ChannelNotFoundError: @@ -373,11 +373,11 @@ def test_create_cid(self): self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = [cid for cid, _ in _channels.list_all()] + before = [cid for cid, _, _ in _channels.list_all()] id1 = _channels.create(REPLACE) id2 = _channels.create(REPLACE) id3 = _channels.create(REPLACE) - after = [cid for cid, _ in _channels.list_all()] + after = [cid for cid, _, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index eada18f99d04db..0c027b17cea68c 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -377,11 +377,11 @@ def common(rch, sch, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is channels.UNBOUND: - extraargs = ', unbound=channels.UNBOUND' + extraargs = ', unbounditems=channels.UNBOUND' elif unbound is channels.UNBOUND_ERROR: - extraargs = ', unbound=channels.UNBOUND_ERROR' + extraargs = ', unbounditems=channels.UNBOUND_ERROR' elif unbound is channels.UNBOUND_REMOVE: - extraargs = ', unbound=channels.UNBOUND_REMOVE' + extraargs = ', unbounditems=channels.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() @@ -454,11 +454,11 @@ def common(rch, sch, unbound=None, presize=0): with self.assertRaises(channels.ChannelEmptyError): rch.recv_nowait() - sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(b'ham', unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 1) interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) self.assertEqual(_channels.get_count(rch.id), 3) - sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(42, unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 4) del interp self.assertEqual(_channels.get_count(rch.id), 2) @@ -484,11 +484,11 @@ def test_send_cleared_with_subinterpreter_mixed(self): _run_output(interp, dedent(f""" from test.support.interpreters import channels sch = channels.SendChannel({sch.id}) - sch.send_nowait(1, unbound=channels.UNBOUND) - sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(1, unbounditems=channels.UNBOUND) + sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) sch.send_nowait(3) - sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(5, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 5) @@ -522,8 +522,8 @@ def test_send_cleared_with_subinterpreter_multiple(self): rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() - sch.send_nowait(2, unbound=channels.UNBOUND) - sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(2, unbounditems=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" from test.support.interpreters import channels @@ -535,21 +535,21 @@ def test_send_cleared_with_subinterpreter_multiple(self): self.assertEqual(_channels.get_count(rch.id), 0) sch.send_nowait(3) _run_output(interp1, dedent(""" - sch.send_nowait(4, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND) # interp closed here - sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(6, unbound=channels.UNBOUND) + sch.send_nowait(5, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbounditems=channels.UNBOUND) """)) _run_output(interp2, dedent(""" - sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(7, unbounditems=channels.UNBOUND_ERROR) # interp closed here - sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) - sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(8, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbounditems=channels.UNBOUND) """)) _run_output(interp1, dedent(""" - sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(10, unbound=channels.UNBOUND) + sch.send_nowait(9, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 10) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 18f83d097eb360..64a2db1230d023 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -9,6 +9,7 @@ _queues = import_helper.import_module('_interpqueues') from test.support import interpreters from test.support.interpreters import queues, _crossinterp +import test._crossinterp_definitions as defs from .utils import _run_output, TestBase as _TestBase @@ -42,7 +43,7 @@ def test_highlevel_reloaded(self): importlib.reload(queues) def test_create_destroy(self): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.destroy(qid) self.assertEqual(get_num_queues(), 0) with self.assertRaises(queues.QueueNotFoundError): @@ -56,7 +57,7 @@ def test_not_destroyed(self): '-c', dedent(f""" import {_queues.__name__} as _queues - _queues.create(2, 0, {REPLACE}) + _queues.create(2, {REPLACE}, -1) """), ) self.assertEqual(stdout, '') @@ -67,13 +68,13 @@ def test_not_destroyed(self): def test_bind_release(self): with self.subTest('typical'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.release(qid) self.assertEqual(get_num_queues(), 0) with self.subTest('bind too much'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -81,7 +82,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('nested'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -89,7 +90,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('release without binding'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) with self.assertRaises(queues.QueueError): _queues.release(qid) @@ -132,13 +133,13 @@ def test_shareable(self): with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2, syncobj=True) + queue1.put(queue2) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4, syncobj=True) + queue1.put(queue4) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -149,7 +150,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5, syncobj=True) + queue1.put(queue5) print(queue5.id) """)) qid = int(out) @@ -198,7 +199,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None, syncobj=True) + queue.put(None) during = queue.empty() queue.get() after = queue.empty() @@ -213,7 +214,7 @@ def test_full(self): queue = queues.create(3) for _ in range(3): actual.append(queue.full()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.full()) for _ in range(3): queue.get() @@ -227,16 +228,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -245,70 +246,32 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create() - for i in range(20): - queue.put(i, **kwds) - actual = [queue.get() for _ in range(20)] + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put(None, **kwds) - queue.put(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1, **kwds) - queue.get() - queue.put(None, **kwds) + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + queue.get() + queue.put(None) def test_put_nowait(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put_nowait(None, **kwds) - queue.put_nowait(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None, **kwds) - queue.get() - queue.put_nowait(None, **kwds) - - def test_put_syncobj(self): - for obj in [ - None, - True, - 10, - 'spam', - b'spam', - (0, 'a'), - ]: - with self.subTest(repr(obj)): - queue = queues.create() - - queue.put(obj, syncobj=True) - obj2 = queue.get() - self.assertEqual(obj2, obj) - - queue.put(obj, syncobj=True) - obj2 = queue.get_nowait() - self.assertEqual(obj2, obj) - - for obj in [ - [1, 2, 3], - {'a': 13, 'b': 17}, - ]: - with self.subTest(repr(obj)): - queue = queues.create() - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj, syncobj=True) + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + queue.get() + queue.put_nowait(None) - def test_put_not_syncobj(self): + def test_put_full_fallback(self): for obj in [ None, True, @@ -323,11 +286,11 @@ def test_put_not_syncobj(self): with self.subTest(repr(obj)): queue = queues.create() - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get() self.assertEqual(obj2, obj) - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get_nowait() self.assertEqual(obj2, obj) @@ -341,24 +304,9 @@ def test_get_nowait(self): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() - def test_put_get_default_syncobj(self): + def test_put_get_full_fallback(self): expected = list(range(20)) - queue = queues.create(syncobj=True) - for methname in ('get', 'get_nowait'): - with self.subTest(f'{methname}()'): - get = getattr(queue, methname) - for i in range(20): - queue.put(i) - actual = [get() for _ in range(20)] - self.assertEqual(actual, expected) - - obj = [1, 2, 3] # lists are not shareable - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj) - - def test_put_get_default_not_syncobj(self): - expected = list(range(20)) - queue = queues.create(syncobj=False) + queue = queues.create() for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): get = getattr(queue, methname) @@ -384,7 +332,7 @@ def test_put_get_same_interpreter(self): with self.subTest(f'{methname}()'): interp.exec(dedent(f""" orig = b'spam' - queue.put(orig, syncobj=True) + queue.put(orig) obj = queue.{methname}() assert obj == orig, 'expected: obj == orig' assert obj is not orig, 'expected: obj is not orig' @@ -399,7 +347,7 @@ def test_put_get_different_interpreters(self): for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): obj1 = b'spam' - queue1.put(obj1, syncobj=True) + queue1.put(obj1) out = _run_output( interp, @@ -416,7 +364,7 @@ def test_put_get_different_interpreters(self): obj2 = b'eggs' print(id(obj2)) assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2, syncobj=True) + queue2.put(obj2) assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) self.assertEqual(len(queues.list_all()), 2) @@ -433,11 +381,11 @@ def common(queue, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is queues.UNBOUND: - extraargs = ', unbound=queues.UNBOUND' + extraargs = ', unbounditems=queues.UNBOUND' elif unbound is queues.UNBOUND_ERROR: - extraargs = ', unbound=queues.UNBOUND_ERROR' + extraargs = ', unbounditems=queues.UNBOUND_ERROR' elif unbound is queues.UNBOUND_REMOVE: - extraargs = ', unbound=queues.UNBOUND_REMOVE' + extraargs = ', unbounditems=queues.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() @@ -447,8 +395,8 @@ def common(queue, unbound=None, presize=0): queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1, syncobj=True{extraargs}) - queue.put(obj2, syncobj=True{extraargs}) + queue.put(obj1{extraargs}) + queue.put(obj2{extraargs}) """)) self.assertEqual(queue.qsize(), presize + 2) @@ -501,11 +449,11 @@ def common(queue, unbound=None, presize=0): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() - queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + queue.put(b'ham', unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 1) interp = common(queue, queues.UNBOUND_REMOVE, 1) self.assertEqual(queue.qsize(), 3) - queue.put(42, unbound=queues.UNBOUND_REMOVE) + queue.put(42, unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 4) del interp self.assertEqual(queue.qsize(), 2) @@ -523,11 +471,11 @@ def test_put_cleared_with_subinterpreter_mixed(self): _run_output(interp, dedent(f""" from test.support.interpreters import queues queue = queues.Queue({queue.id}) - queue.put(1, syncobj=True, unbound=queues.UNBOUND) - queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(3, syncobj=True) - queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(5, syncobj=True, unbound=queues.UNBOUND) + queue.put(1, unbounditems=queues.UNBOUND) + queue.put(2, unbounditems=queues.UNBOUND_ERROR) + queue.put(3) + queue.put(4, unbounditems=queues.UNBOUND_REMOVE) + queue.put(5, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 5) @@ -555,13 +503,13 @@ def test_put_cleared_with_subinterpreter_multiple(self): interp1 = interpreters.create() interp2 = interpreters.create() - queue.put(1, syncobj=True) + queue.put(1) _run_output(interp1, dedent(f""" from test.support.interpreters import queues queue = queues.Queue({queue.id}) obj1 = queue.get() - queue.put(2, syncobj=True, unbound=queues.UNBOUND) - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(2, unbounditems=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" from test.support.interpreters import queues @@ -572,21 +520,21 @@ def test_put_cleared_with_subinterpreter_multiple(self): self.assertEqual(queue.qsize(), 0) queue.put(3) _run_output(interp1, dedent(""" - queue.put(4, syncobj=True, unbound=queues.UNBOUND) + queue.put(4, unbounditems=queues.UNBOUND) # interp closed here - queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(6, syncobj=True, unbound=queues.UNBOUND) + queue.put(5, unbounditems=queues.UNBOUND_REMOVE) + queue.put(6, unbounditems=queues.UNBOUND) """)) _run_output(interp2, dedent(""" - queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(7, unbounditems=queues.UNBOUND_ERROR) # interp closed here - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(8, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_ERROR) + queue.put(obj2, unbounditems=queues.UNBOUND_REMOVE) + queue.put(8, unbounditems=queues.UNBOUND) """)) _run_output(interp1, dedent(""" - queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(10, syncobj=True, unbound=queues.UNBOUND) + queue.put(9, unbounditems=queues.UNBOUND_REMOVE) + queue.put(10, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 10) @@ -642,12 +590,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj, syncobj=True) + queue2.put(obj) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig, syncobj=True) + queue1.put(orig) obj = queue2.get() t.join() diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index f9fa1dab291056..0ab553190001bd 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -20,9 +20,11 @@ #endif #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -523,7 +525,7 @@ typedef struct _channelitem { int64_t interpid; _PyXIData_t *data; _waiting_t *waiting; - int unboundop; + unboundop_t unboundop; struct _channelitem *next; } _channelitem; @@ -536,7 +538,7 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -583,7 +585,7 @@ _channelitem_clear(_channelitem *item) static _channelitem * _channelitem_new(int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { @@ -694,7 +696,7 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { @@ -798,7 +800,7 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - int unboundop; + unboundop_t unboundop; _channelitem_popped(item, p_data, p_waiting, &unboundop); } @@ -1083,16 +1085,18 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; - struct { - int unboundop; + struct _channeldefaults { + unboundop_t unboundop; + xidata_fallback_t fallback; } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex, int unboundop) +_channel_new(PyThread_type_lock mutex, struct _channeldefaults defaults) { + assert(check_unbound(defaults.unboundop)); _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { return NULL; @@ -1109,7 +1113,7 @@ _channel_new(PyThread_type_lock mutex, int unboundop) GLOBAL_FREE(chan); return NULL; } - chan->defaults.unboundop = unboundop; + chan->defaults = defaults; chan->open = 1; chan->closing = NULL; return chan; @@ -1130,7 +1134,7 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyXIData_t *data, _waiting_t *waiting, int unboundop) + _PyXIData_t *data, _waiting_t *waiting, unboundop_t unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1611,7 +1615,7 @@ _channels_release_cid_object(_channels *channels, int64_t cid) struct channel_id_and_info { int64_t id; - int unboundop; + struct _channeldefaults defaults; }; static struct channel_id_and_info * @@ -1628,7 +1632,7 @@ _channels_list_all(_channels *channels, int64_t *count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i] = (struct channel_id_and_info){ .id = ref->cid, - .unboundop = ref->chan->defaults.unboundop, + .defaults = ref->chan->defaults, }; } *count = channels->numopen; @@ -1714,13 +1718,13 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels, int unboundop) +channel_create(_channels *channels, struct _channeldefaults defaults) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_CHANNEL_MUTEX_INIT; } - _channel_state *chan = _channel_new(mutex, unboundop); + _channel_state *chan = _channel_new(mutex, defaults); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1752,7 +1756,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop, xidata_fallback_t fallback) { PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -1779,7 +1783,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; @@ -1823,7 +1827,8 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - int unboundop, PY_TIMEOUT_T timeout) + unboundop_t unboundop, PY_TIMEOUT_T timeout, + xidata_fallback_t fallback) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1834,7 +1839,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting, unboundop); + int res = channel_send(channels, cid, obj, &waiting, unboundop, fallback); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -2005,6 +2010,20 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +channel_get_defaults(_channels *channels, int64_t cid, struct _channeldefaults *defaults) +{ + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(channels, cid, &mutex, &channel); + if (err != 0) { + return err; + } + *defaults = channel->defaults; + PyThread_release_lock(mutex); + return 0; +} + static int _channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) { @@ -2881,20 +2900,27 @@ clear_interpreter(void *data) static PyObject * channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"unboundop", NULL}; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, - &unboundop)) + static char *kwlist[] = {"unboundop", "fallback", NULL}; + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:create", kwlist, + &unboundarg, &fallbackarg)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _channeldefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) + { + return NULL; + } + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t cid = channel_create(&_globals.channels, unboundop); + int64_t cid = channel_create(&_globals.channels, defaults); if (cid < 0) { (void)handle_channel_error(-1, self, cid); return NULL; @@ -2987,7 +3013,9 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } assert(cidobj != NULL); - PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + PyObject *item = Py_BuildValue("Oii", cidobj, + cur->defaults.unboundop, + cur->defaults.fallback); Py_DECREF(cidobj); if (item == NULL) { Py_SETREF(ids, NULL); @@ -3075,40 +3103,54 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; + int unboundarg = -1; + int fallbackarg = -1; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&O|ii$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); - return NULL; - } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); + err = channel_send_wait( + &_globals.channels, cid, obj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, obj, NULL, unboundop, fallback); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -3126,32 +3168,44 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; - int blocking = 1; + int unboundarg = -1; + int fallbackarg = -1; + int blocking = -1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|i$pO:channel_send_buffer", kwlist, + "O&O|ii$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) { - return NULL; - } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) + { return NULL; } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } PyObject *tempobj = PyMemoryView_FromObject(obj); if (tempobj == NULL) { @@ -3162,10 +3216,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) int err = 0; if (blocking) { err = channel_send_wait( - &_globals.channels, cid, tempobj, unboundop, timeout); + &_globals.channels, cid, tempobj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, tempobj, NULL, unboundop, fallback); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3197,7 +3252,7 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int unboundop = 0; + unboundop_t unboundop = 0; int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. @@ -3388,17 +3443,14 @@ channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t cid = cid_data.cid; - PyThread_type_lock mutex = NULL; - _channel_state *channel = NULL; - int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + struct _channeldefaults defaults; + int err = channel_get_defaults(&_globals.channels, cid, &defaults); if (handle_channel_error(err, self, cid)) { return NULL; } - int unboundop = channel->defaults.unboundop; - PyThread_release_lock(mutex); - PyObject *defaults = Py_BuildValue("i", unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 209fcdfd0cd01e..816285c9eff44a 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -9,9 +9,11 @@ #include "pycore_crossinterp.h" // _PyXIData_t #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -401,14 +403,13 @@ typedef struct _queueitem { meaning the interpreter has been destroyed. */ int64_t interpid; _PyXIData_t *data; - int fmt; - int unboundop; + unboundop_t unboundop; struct _queueitem *next; } _queueitem; static void _queueitem_init(_queueitem *item, - int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) + int64_t interpid, _PyXIData_t *data, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -422,7 +423,6 @@ _queueitem_init(_queueitem *item, *item = (_queueitem){ .interpid = interpid, .data = data, - .fmt = fmt, .unboundop = unboundop, }; } @@ -446,14 +446,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) +_queueitem_new(int64_t interpid, _PyXIData_t *data, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, interpid, data, fmt, unboundop); + _queueitem_init(item, interpid, data, unboundop); return item; } @@ -476,10 +476,9 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) + _PyXIData_t **p_data, unboundop_t *p_unboundop) { *p_data = item->data; - *p_fmt = item->fmt; *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; @@ -527,16 +526,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; - struct { - int fmt; + struct _queuedefaults { + xidata_fallback_t fallback; int unboundop; } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) +_queue_init(_queue *queue, Py_ssize_t maxsize, struct _queuedefaults defaults) { - assert(check_unbound(unboundop)); + assert(check_unbound(defaults.unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -547,10 +546,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) .items = { .maxsize = maxsize, }, - .defaults = { - .fmt = fmt, - .unboundop = unboundop, - }, + .defaults = defaults, }; return 0; } @@ -631,8 +627,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, - int fmt, int unboundop) +_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -648,7 +643,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); + _queueitem *item = _queueitem_new(interpid, data, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -668,8 +663,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, } static int -_queue_next(_queue *queue, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) +_queue_next(_queue *queue, _PyXIData_t **p_data, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -688,7 +682,7 @@ _queue_next(_queue *queue, } queue->items.count -= 1; - _queueitem_popped(item, p_data, p_fmt, p_unboundop); + _queueitem_popped(item, p_data, p_unboundop); _queue_unlock(queue); return 0; @@ -1035,8 +1029,7 @@ _queues_decref(_queues *queues, int64_t qid) struct queue_id_and_info { int64_t id; - int fmt; - int unboundop; + struct _queuedefaults defaults; }; static struct queue_id_and_info * @@ -1053,8 +1046,7 @@ _queues_list_all(_queues *queues, int64_t *p_count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i].id = ref->qid; assert(ref->queue != NULL); - ids[i].fmt = ref->queue->defaults.fmt; - ids[i].unboundop = ref->queue->defaults.unboundop; + ids[i].defaults = ref->queue->defaults; } *p_count = queues->count; @@ -1090,13 +1082,14 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) +queue_create(_queues *queues, Py_ssize_t maxsize, + struct _queuedefaults defaults) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize, fmt, unboundop); + int err = _queue_init(queue, maxsize, defaults); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; @@ -1125,7 +1118,8 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) +queue_put(_queues *queues, int64_t qid, PyObject *obj, unboundop_t unboundop, + xidata_fallback_t fallback) { PyThreadState *tstate = PyThreadState_Get(); @@ -1138,27 +1132,27 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) assert(queue != NULL); // Convert the object to cross-interpreter data. - _PyXIData_t *data = _PyXIData_New(); - if (data == NULL) { + _PyXIData_t *xidata = _PyXIData_New(); + if (xidata == NULL) { _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetXIDataNoFallback(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) { _queue_unmark_waiter(queue, queues->mutex); - GLOBAL_FREE(data); + GLOBAL_FREE(xidata); return -1; } - assert(_PyXIData_INTERPID(data) == + assert(_PyXIData_INTERPID(xidata) == PyInterpreterState_GetID(tstate->interp)); // Add the data to the queue. int64_t interpid = -1; // _queueitem_init() will set it. - int res = _queue_add(queue, interpid, data, fmt, unboundop); + int res = _queue_add(queue, interpid, xidata, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: - (void)_release_xid_data(data, 0); - GLOBAL_FREE(data); + (void)_release_xid_data(xidata, 0); + GLOBAL_FREE(xidata); return res; } @@ -1169,7 +1163,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) // XXX Support a "wait" mutex? static int queue_get(_queues *queues, int64_t qid, - PyObject **res, int *p_fmt, int *p_unboundop) + PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1185,7 +1179,7 @@ queue_get(_queues *queues, int64_t qid, // Pop off the next item from the queue. _PyXIData_t *data = NULL; - err = _queue_next(queue, &data, p_fmt, p_unboundop); + err = _queue_next(queue, &data, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1216,6 +1210,20 @@ queue_get(_queues *queues, int64_t qid, return 0; } +static int +queue_get_defaults(_queues *queues, int64_t qid, + struct _queuedefaults *p_defaults) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + *p_defaults = queue->defaults; + _queue_unmark_waiter(queue, queues->mutex); + return 0; +} + static int queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) { @@ -1474,22 +1482,28 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; + static char *kwlist[] = {"maxsize", "unboundop", "fallback", NULL}; Py_ssize_t maxsize; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, - &maxsize, &fmt, &unboundop)) + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|ii:create", kwlist, + &maxsize, &unboundarg, &fallbackarg)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _queuedefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) + { + return NULL; + } + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); + int64_t qid = queue_create(&_globals.queues, maxsize, defaults); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1511,7 +1525,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create(maxsize, fmt, unboundop) -> qid\n\ +"create(maxsize, unboundop, fallback) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ It is a new reference as though bind() had been called on the queue.\n\ @@ -1560,8 +1574,9 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, - cur->unboundop); + PyObject *item = Py_BuildValue("Lii", cur->id, + cur->defaults.unboundop, + cur->defaults.fallback); if (item == NULL) { Py_SETREF(ids, NULL); break; @@ -1575,34 +1590,44 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [(qid, fmt)]\n\ +"list_all() -> [(qid, unboundop, fallback)]\n\ \n\ Return the list of IDs for all queues.\n\ -Each corresponding default format is also included."); +Each corresponding default unbound op and fallback is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; + static char *kwlist[] = {"qid", "obj", "unboundop", "fallback", NULL}; qidarg_converter_data qidarg = {0}; PyObject *obj; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, - qidarg_converter, &qidarg, &obj, &fmt, - &unboundop)) + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|ii$p:put", kwlist, + qidarg_converter, &qidarg, &obj, + &unboundarg, &fallbackarg)) { return NULL; } int64_t qid = qidarg.id; - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _queuedefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = queue_get_defaults(&_globals.queues, qid, &defaults); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { return NULL; } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); + int err = queue_put(&_globals.queues, qid, obj, unboundop, fallback); // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; @@ -1612,7 +1637,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj, fmt)\n\ +"put(qid, obj)\n\ \n\ Add the object's data to the queue."); @@ -1628,27 +1653,26 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) int64_t qid = qidarg.id; PyObject *obj = NULL; - int fmt = 0; int unboundop = 0; - int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); + int err = queue_get(&_globals.queues, qid, &obj, &unboundop); // This is the only place that raises QueueEmpty. if (handle_queue_error(err, self, qid)) { return NULL; } if (obj == NULL) { - return Py_BuildValue("Oii", Py_None, fmt, unboundop); + return Py_BuildValue("Oi", Py_None, unboundop); } - PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); + PyObject *res = Py_BuildValue("OO", obj, Py_None); Py_DECREF(obj); return res; } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid) -> (obj, fmt)\n\ +"get(qid) -> (obj, unboundop)\n\ \n\ Return a new object from the data at the front of the queue.\n\ -The object's format is also returned.\n\ +The unbound op is also returned.\n\ \n\ If there is nothing to receive then raise QueueEmpty."); @@ -1748,17 +1772,14 @@ queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t qid = qidarg.id; - _queue *queue = NULL; - int err = _queues_lookup(&_globals.queues, qid, &queue); + struct _queuedefaults defaults; + int err = queue_get_defaults(&_globals.queues, qid, &defaults); if (handle_queue_error(err, self, qid)) { return NULL; } - int fmt = queue->defaults.fmt; - int unboundop = queue->defaults.unboundop; - _queue_unmark_waiter(queue, _globals.queues.mutex); - PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index d73cbca36359c7..40fd51d752e324 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -39,10 +39,37 @@ _get_interpid(_PyXIData_t *data) } +#ifdef HAS_FALLBACK +static int +resolve_fallback(int arg, xidata_fallback_t dflt, + xidata_fallback_t *p_fallback) +{ + if (arg < 0) { + *p_fallback = dflt; + return 0; + } + xidata_fallback_t fallback; + if (arg == _PyXIDATA_XIDATA_ONLY) { + fallback =_PyXIDATA_XIDATA_ONLY; + } + else if (arg == _PyXIDATA_FULL_FALLBACK) { + fallback = _PyXIDATA_FULL_FALLBACK; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported fallback %d", arg); + return -1; + } + *p_fallback = fallback; + return 0; +} +#endif + + /* unbound items ************************************************************/ #ifdef HAS_UNBOUND_ITEMS +typedef int unboundop_t; #define UNBOUND_REMOVE 1 #define UNBOUND_ERROR 2 #define UNBOUND_REPLACE 3 @@ -53,6 +80,7 @@ _get_interpid(_PyXIData_t *data) // object is released but the underlying data is copied (with the "raw" // allocator) and used when the item is popped off the queue. +#ifndef NDEBUG static int check_unbound(int unboundop) { @@ -65,5 +93,31 @@ check_unbound(int unboundop) return 0; } } +#endif + +static int +resolve_unboundop(int arg, unboundop_t dflt, unboundop_t *p_unboundop) +{ + if (arg < 0) { + *p_unboundop = dflt; + return 0; + } + unboundop_t op; + if (arg == UNBOUND_REMOVE) { + op = UNBOUND_REMOVE; + } + else if (arg == UNBOUND_ERROR) { + op = UNBOUND_ERROR; + } + else if (arg == UNBOUND_REPLACE) { + op = UNBOUND_REPLACE; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported unboundop %d", arg); + return -1; + } + *p_unboundop = op; + return 0; +} #endif diff --git a/Python/crossinterp.c b/Python/crossinterp.c index dc67de4a40849d..6681b969183925 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1839,6 +1839,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); + // XXX Use _PyObject_GetXIDataWithFallback()? if (_PyObject_GetXIDataNoFallback(tstate, value, item->xidata) != 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; From 7e7391810fa1e8e378d6073c57914bc79d4e9295 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 15:52:37 +0200 Subject: [PATCH 200/713] [3.14] gh-134370: Added clarification on instance annotations (GH-134387) (#134509) gh-134370: Added clarification on instance annotations (GH-134387) Instances of classes cannot have annotations, however sometimes they will erroneously have the __annotations__ attribute (cherry picked from commit bd4046f4f869039a1a2ebe2d1d18bfbc2a2951b6) Co-authored-by: Alex Kautz --- Doc/howto/annotations.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 78f3704ba5d000..d7deb6c6bc1768 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -248,4 +248,9 @@ quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or :func:`inspect.get_annotations` on Python 3.10+. On earlier versions of Python, you can avoid these bugs by accessing the annotations from the class's :attr:`~type.__dict__` -(e.g., ``cls.__dict__.get('__annotations__', None)``). +(for example, ``cls.__dict__.get('__annotations__', None)``). + +In some versions of Python, instances of classes may have an ``__annotations__`` +attribute. However, this is not supported functionality. If you need the +annotations of an instance, you can use :func:`type` to access its class +(for example, ``annotationlib.get_annotations(type(myinstance))`` on Python 3.14+). From ced49a196f0e1ea06ba892b875f18e7b11ed26b7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 16:14:53 +0200 Subject: [PATCH 201/713] [3.14] gh-134323: Fix the new `threading.RLock.locked` method (GH-134368) (#134510) gh-134323: Fix the new `threading.RLock.locked` method (GH-134368) (cherry picked from commit 3effede97cc13fc0c5ab5dcde26cc319f388e84c) Co-authored-by: Duprat Co-authored-by: Kumar Aditya --- Lib/test/lock_tests.py | 18 ++++++++++++++++++ Lib/threading.py | 4 ++-- ...5-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst | 1 + Modules/_threadmodule.c | 10 ++++++++-- 4 files changed, 29 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 009e04e9c0b522..0e8c75ff5c4994 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -365,6 +365,24 @@ def test_locked(self): lock.release() self.assertFalse(lock.locked()) + def test_locked_with_2threads(self): + # see gh-134323: check that a rlock which + # is acquired in a different thread, + # is still locked in the main thread. + result = [] + rlock = self.locktype() + self.assertFalse(rlock.locked()) + def acquire(): + result.append(rlock.locked()) + rlock.acquire() + result.append(rlock.locked()) + + with Bunch(acquire, 1): + pass + self.assertTrue(rlock.locked()) + self.assertFalse(result[0]) + self.assertTrue(result[1]) + def test_release_save_unacquired(self): # Cannot _release_save an unacquired lock lock = self.locktype() diff --git a/Lib/threading.py b/Lib/threading.py index fa290d17c635ab..2a65f9a7aa3028 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -165,7 +165,7 @@ def __repr__(self): except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( - "locked" if self._block.locked() else "unlocked", + "locked" if self.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, @@ -244,7 +244,7 @@ def __exit__(self, t, v, tb): def locked(self): """Return whether this object is locked.""" - return self._count > 0 + return self._block.locked() # Internal methods used by condition variables diff --git a/Misc/NEWS.d/next/Library/2025-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst b/Misc/NEWS.d/next/Library/2025-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst new file mode 100644 index 00000000000000..7982b52f77a172 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst @@ -0,0 +1 @@ +Fix the :meth:`threading.RLock.locked` method. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 9776a32755db68..8bd59fffa66f88 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -1011,6 +1011,11 @@ rlock_traverse(PyObject *self, visitproc visit, void *arg) return 0; } +static int +rlock_locked_impl(rlockobject *self) +{ + return PyMutex_IsLocked(&self->lock.mutex); +} static void rlock_dealloc(PyObject *self) @@ -1100,7 +1105,7 @@ static PyObject * rlock_locked(PyObject *op, PyObject *Py_UNUSED(ignored)) { rlockobject *self = rlockobject_CAST(op); - int is_locked = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + int is_locked = rlock_locked_impl(self); return PyBool_FromLong(is_locked); } @@ -1202,10 +1207,11 @@ rlock_repr(PyObject *op) { rlockobject *self = rlockobject_CAST(op); PyThread_ident_t owner = self->lock.thread; + int locked = rlock_locked_impl(self); size_t count = self->lock.level + 1; return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", - owner ? "locked" : "unlocked", + locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, owner, count, self); } From c31b25c705fffc42f955def6b8f4f7d2bc96916c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 16:31:26 +0200 Subject: [PATCH 202/713] [3.14] gh-133740: Fix locale.nl_langinfo(ALT_DIGITS) (GH-134468) (#134512) gh-133740: Fix locale.nl_langinfo(ALT_DIGITS) (GH-134468) Set the LC_CTYPE locale to the LC_TIME locale even if nl_langinfo(ALT_DIGITS) result is ASCII. The result is a list separated by NUL characters and the code only checks the first list item which can be ASCII whereas following items are non-ASCII. Fix test__locale for the uk_UA locale on RHEL 7. (cherry picked from commit 899c7dc283cb899fdfd79fb479b38352f48e454a) Co-authored-by: Victor Stinner Co-authored-by: Serhiy Storchaka --- Modules/_localemodule.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index ad618398d5b824..c1f56008b7c49e 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -692,7 +692,17 @@ _locale_nl_langinfo_impl(PyObject *module, int item) result = result != NULL ? result : ""; char *oldloc = NULL; if (langinfo_constants[i].category != LC_CTYPE - && !is_all_ascii(result) + && ( +#ifdef __GLIBC__ + // gh-133740: Always change the locale for ALT_DIGITS and ERA +# ifdef ALT_DIGITS + item == ALT_DIGITS || +# endif +# ifdef ERA + item == ERA || +# endif +#endif + !is_all_ascii(result)) && change_locale(langinfo_constants[i].category, &oldloc) < 0) { return NULL; From bbf8048c0f9c12b606da2e3e7b70e47e9f93f506 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 17:07:11 +0200 Subject: [PATCH 203/713] [3.14] gh-132775: Use _PyCode GetScriptXIData() (gh-134515) (cherry picked from commit 09e72cf091d, AKA gh-134511) Co-authored-by: Eric Snow --- Lib/test/support/interpreters/channels.py | 2 +- Lib/test/test__interpreters.py | 25 +- Lib/test/test_interpreters/test_api.py | 23 +- Modules/_interpretersmodule.c | 294 ++++++---------------- 4 files changed, 104 insertions(+), 240 deletions(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 7a2bd7d63f808f..3b6e0f0effd969 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -69,7 +69,7 @@ def list_all(): if not hasattr(send, '_unboundop'): send._set_unbound(unboundop) else: - assert send._unbound[0] == op + assert send._unbound[0] == unboundop channels.append(chan) return channels diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 63fdaad8de7ef5..ad3ebbfdff64a7 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,13 +474,15 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = "expected 'shared' to be a dict" + msg = r'_interpreters.exec\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) + msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) + msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) @@ -952,7 +954,8 @@ def test_invalid_syntax(self): """) with self.subTest('script'): - self.assert_run_failed(SyntaxError, script) + with self.assertRaises(SyntaxError): + _interpreters.run_string(self.id, script) with self.subTest('module'): modname = 'spam_spam_spam' @@ -1019,12 +1022,19 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') + failed = None def f(): - _interpreters.set___main___attrs(self.id, dict(w=w)) - _interpreters.run_func(self.id, script) + nonlocal failed + try: + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) + except Exception as exc: + failed = exc t = threading.Thread(target=f) t.start() t.join() + if failed: + raise Exception from failed with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1053,19 +1063,16 @@ def test_closure(self): spam = True def script(): assert spam - - with self.assertRaises(TypeError): + with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) - # XXX This hasn't been fixed yet. - @unittest.expectedFailure def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) - @unittest.skip("we're not quite there yet") +# @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 1e2d572b1cbb81..165949167ceba8 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -839,9 +839,16 @@ def test_bad_script(self): interp.exec(10) def test_bytes_for_script(self): + r, w = self.pipe() + RAN = b'R' + DONE = b'D' interp = interpreters.create() - with self.assertRaises(TypeError): - interp.exec(b'print("spam")') + interp.exec(f"""if True: + import os + os.write({w}, {RAN!r}) + """) + os.write(w, DONE) + self.assertEqual(os.read(r, 1), RAN) def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -1010,8 +1017,6 @@ def test_call(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1036,6 +1041,8 @@ def test_call(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): with self.assertRaises(Exception): @@ -1051,8 +1058,6 @@ def test_call_in_thread(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1079,6 +1084,8 @@ def test_call_in_thread(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): if args or kwargs: @@ -1618,8 +1625,8 @@ def test_exec(self): def test_call(self): with self.subTest('no args'): interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_return_shareable) - self.assertIs(exc, None) + with self.assertRaises(ValueError): + _interpreters.call(interpid, call_func_return_shareable) with self.subTest('uncaught exception'): interpid = _interpreters.create() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 91cd92806206be..7f84b38a70e127 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -9,7 +9,6 @@ #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() -#include "pycore_function.h" // _PyFunction_VerifyStateless() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -361,81 +360,6 @@ _get_current_xibufferview_type(void) } -/* Python code **************************************************************/ - -static const char * -check_code_str(PyUnicodeObject *text) -{ - assert(text != NULL); - if (PyUnicode_GET_LENGTH(text) == 0) { - return "too short"; - } - - // XXX Verify that it parses? - - return NULL; -} - -#ifndef NDEBUG -static int -code_has_args(PyCodeObject *code) -{ - assert(code != NULL); - return (code->co_argcount > 0 - || code->co_posonlyargcount > 0 - || code->co_kwonlyargcount > 0 - || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)); -} -#endif - -#define RUN_TEXT 1 -#define RUN_CODE 2 - -static const char * -get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) -{ - const char *codestr = NULL; - Py_ssize_t len = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - - if (PyUnicode_Check(arg)) { - assert(PyUnicode_Check(arg) - && (check_code_str((PyUnicodeObject *)arg) == NULL)); - codestr = PyUnicode_AsUTF8AndSize(arg, &len); - if (codestr == NULL) { - return NULL; - } - if (strlen(codestr) != (size_t)len) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - flags = RUN_TEXT; - } - else { - assert(PyCode_Check(arg)); - assert(_PyCode_VerifyStateless( - PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0); - assert(!code_has_args((PyCodeObject *)arg)); - flags = RUN_CODE; - - // Serialize the code object. - bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); - if (bytes_obj == NULL) { - return NULL; - } - codestr = PyBytes_AS_STRING(bytes_obj); - len = PyBytes_GET_SIZE(bytes_obj); - } - - *flags_p = flags; - *bytes_p = bytes_obj; - *len_p = len; - return codestr; -} - - /* interpreter-specific code ************************************************/ static int @@ -499,22 +423,14 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) static int -_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) +_run_script(_PyXIData_t *script, PyObject *ns) { - PyObject *result = NULL; - if (flags & RUN_TEXT) { - result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); - } - else if (flags & RUN_CODE) { - PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); - if (code != NULL) { - result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); - } - } - else { - Py_UNREACHABLE(); + PyObject *code = _PyXIData_NewObject(script); + if (code == NULL) { + return -1; } + PyObject *result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); if (result == NULL) { return -1; } @@ -523,17 +439,16 @@ _run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) } static int -_run_in_interpreter(PyInterpreterState *interp, - const char *codestr, Py_ssize_t codestrlen, - PyObject *shareables, int flags, +_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, + _PyXIData_t *script, PyObject *shareables, PyObject **p_excinfo) { - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); _PyXI_session session = {0}; // Prep and switch interpreters. if (_PyXI_Enter(&session, interp, shareables) < 0) { - if (PyErr_Occurred()) { + if (_PyErr_Occurred(tstate)) { // If an error occured at this step, it means that interp // was not prepared and switched. return -1; @@ -548,7 +463,7 @@ _run_in_interpreter(PyInterpreterState *interp, } // Run the script. - int res = _run_script(session.main_ns, codestr, codestrlen, flags); + int res = _run_script(script, session.main_ns); // Clean up and switch back. _PyXI_Exit(&session); @@ -935,104 +850,23 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static PyUnicodeObject * -convert_script_arg(PyThreadState *tstate, - PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - PyUnicodeObject *str = NULL; - if (PyUnicode_CheckExact(arg)) { - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else if (PyUnicode_Check(arg)) { - // XXX str = PyUnicode_FromObject(arg); - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_str(str); - if (err != NULL) { - Py_DECREF(str); - _PyErr_Format(tstate, PyExc_ValueError, - "%.200s(): bad script text (%s)", fname, err); - return NULL; - } - - return str; -} - -static PyCodeObject * -convert_code_arg(PyThreadState *tstate, - PyObject *arg, const char *fname, const char *displayname, - const char *expected) +static void +unwrap_not_shareable(PyThreadState *tstate) { - PyObject *cause; - PyCodeObject *code = NULL; - if (PyFunction_Check(arg)) { - // For now we allow globals, so we can't use - // _PyFunction_VerifyStateless(). - PyObject *codeobj = PyFunction_GetCode(arg); - if (_PyCode_VerifyStateless( - tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) { - goto chained; - } - code = (PyCodeObject *)Py_NewRef(codeobj); + PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); + if (!_PyErr_ExceptionMatches(tstate, exctype)) { + return; } - else if (PyCode_Check(arg)) { - if (_PyCode_VerifyStateless( - tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) { - goto chained; - } - code = (PyCodeObject *)Py_NewRef(arg); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; } else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; + assert(PyException_GetContext(exc) == NULL); } - - return code; - -chained: - cause = _PyErr_GetRaisedException(tstate); - assert(cause != NULL); - _PyArg_BadArgument(fname, displayname, expected, arg); - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyException_SetCause(exc, cause); _PyErr_SetRaisedException(tstate, exc); - return NULL; -} - -static int -_interp_exec(PyObject *self, PyInterpreterState *interp, - PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) -{ - if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { - PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); - return -1; - } - - // Extract code. - Py_ssize_t codestrlen = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - const char *codestr = get_code_str(code_arg, - &codestrlen, &bytes_obj, &flags); - if (codestr == NULL) { - return -1; - } - - // Run the code in the interpreter. - int res = _run_in_interpreter(interp, codestr, codestrlen, - shared_arg, flags, p_excinfo); - Py_XDECREF(bytes_obj); - if (res < 0) { - return -1; - } - - return 0; } static PyObject * @@ -1045,8 +879,9 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" FUNCNAME, kwlist, - &id, &code, &shared, &restricted)) + "OO|O!$p:" FUNCNAME, kwlist, + &id, &code, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1058,22 +893,17 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - const char *expected = "a string, a function, or a code object"; - if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME, - "argument 2", expected); - } - else { - code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME, - "argument 2", expected); - } - if (code == NULL) { + // We don't need the script to be "pure", which means it can use + // global variables. They will be resolved against __main__. + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate); return NULL; } PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, shared, &excinfo); - Py_DECREF(code); + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXIData_Release(&xidata); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1109,8 +939,9 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O$p:" FUNCNAME, kwlist, - &id, &script, &shared, &restricted)) + "OU|O!$p:" FUNCNAME, kwlist, + &id, &script, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1122,15 +953,20 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME, - "argument 2", "a string"); - if (script == NULL) { + if (PyFunction_Check(script) || PyCode_Check(script)) { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); + return NULL; + } + + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { + unwrap_not_shareable(tstate); return NULL; } PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, script, shared, &excinfo); - Py_DECREF(script); + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXIData_Release(&xidata); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1156,8 +992,9 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" FUNCNAME, kwlist, - &id, &func, &shared, &restricted)) + "OO|O!$p:" FUNCNAME, kwlist, + &id, &func, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1169,16 +1006,29 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME, - "argument 2", - "a function or a code object"); - if (code == NULL) { + // We don't worry about checking globals. They will be resolved + // against __main__. + PyObject *code; + if (PyFunction_Check(func)) { + code = PyFunction_GET_CODE(func); + } + else if (PyCode_Check(func)) { + code = func; + } + else { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); + return NULL; + } + + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate); return NULL; } PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); - Py_DECREF(code); + int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); + _PyXIData_Release(&xidata); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1231,15 +1081,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME, - "argument 2", "a function"); - if (code == NULL) { + _PyXIData_t xidata = {0}; + if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) { + unwrap_not_shareable(tstate); return NULL; } PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, NULL, &excinfo); - Py_DECREF(code); + int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo); + _PyXIData_Release(&xidata); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; From 068d570be84eab2ef271204e7ba9dd37391791d4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 19:12:21 +0200 Subject: [PATCH 204/713] [3.14] gh-134322: Fix `repr(threading.RLock)` (GH-134389) (#134528) gh-134322: Fix `repr(threading.RLock)` (GH-134389) Fix the `__repr__` value of `threading.RLock` from `_thread` module, when just created. (cherry picked from commit fade146cfb1616ad7b3b918bedb86756dedf79e6) Co-authored-by: Duprat --- Lib/test/lock_tests.py | 20 ++++++++++++++++++++ Lib/test/test_importlib/test_locks.py | 1 + Modules/_threadmodule.c | 8 +++++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 0e8c75ff5c4994..fb11f4828957ff 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -332,6 +332,26 @@ class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("lock.thread; int locked = rlock_locked_impl(self); - size_t count = self->lock.level + 1; + size_t count; + if (locked) { + count = self->lock.level + 1; + } + else { + count = 0; + } return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", locked ? "locked" : "unlocked", From 04e2dd65138b9114b705890bab5671f80c541cf8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 19:12:52 +0200 Subject: [PATCH 205/713] [3.14] gh-132775: Make _PyXI_session Opaque (gh-134522) This is mostly a refactor to clean things up a bit, most notably the "XI namespace" code. Making the session opaque requires adding the following internal-only functions: * _PyXI_NewSession() * _PyXI_FreeSession() * _PyXI_GetMainNamespace() (cherry picked from commit 4a4ac3ab4d, gh-134452) Co-authored-by: Eric Snow --- Doc/data/python3.14.abi | 902 +++++++++++++------------- Include/internal/pycore_crossinterp.h | 58 +- Modules/_interpretersmodule.c | 41 +- Python/crossinterp.c | 723 ++++++++++++--------- 4 files changed, 896 insertions(+), 828 deletions(-) diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index decb2ce3c6f39e..64f4d473b72def 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -1195,11 +1195,13 @@ + + @@ -1510,19 +1512,18 @@ - - - + + - + @@ -4246,17 +4247,17 @@ - + - - - + + + - - - + + + @@ -7099,7 +7100,7 @@ - + @@ -7349,7 +7350,7 @@ - + @@ -26915,73 +26916,59 @@ - - - + + - + + + + - + - - - - + - + + + + - + - + - - - - + - - - - - - - - - - - - - - - - + + - - - - - - - - - - - - + + + + + + + + + + + + + + - @@ -27194,75 +27181,66 @@ - + - + - + - + - + - - - - - - - - - - - - - + + - - - - - + + + - - + + - - + + - - - - + + + + - - + + - - - - - + + + + + + + + + - - - - + + + + @@ -27298,11 +27276,11 @@ - + - + @@ -27321,7 +27299,7 @@ - + @@ -27478,15 +27456,15 @@ - + - + - + @@ -27494,11 +27472,11 @@ - - - - - + + + + + @@ -27542,7 +27520,7 @@ - + @@ -27613,7 +27591,7 @@ - + @@ -27687,9 +27665,9 @@ - + - + @@ -27729,17 +27707,17 @@ - + - + - - - - + + + + @@ -27760,7 +27738,7 @@ - + @@ -27774,7 +27752,7 @@ - + @@ -27782,15 +27760,15 @@ - - - - - - - - - + + + + + + + + + @@ -27850,7 +27828,7 @@ - + @@ -27885,15 +27863,15 @@ - + - + - + @@ -27983,8 +27961,8 @@ - - + + @@ -27996,8 +27974,8 @@ - - + + @@ -28018,7 +27996,7 @@ - + @@ -28031,7 +28009,7 @@ - + @@ -28040,22 +28018,22 @@ - + - - + + - - + + @@ -28072,7 +28050,7 @@ - + @@ -28081,7 +28059,7 @@ - + @@ -28089,24 +28067,24 @@ - + - + - + - + - + - + @@ -28116,9 +28094,9 @@ - - - + + + @@ -28126,48 +28104,48 @@ - + - - - + + + - + - + - + - + - + - + - + - - - + + + @@ -28198,7 +28176,7 @@ - + @@ -28335,7 +28313,7 @@ - + @@ -28347,11 +28325,11 @@ - + - - + + @@ -28371,8 +28349,8 @@ - - + + @@ -28392,27 +28370,27 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + - - + + - - + + - + @@ -28480,13 +28458,13 @@ - + - + @@ -28511,21 +28489,21 @@ - + - + - + - + @@ -28534,7 +28512,7 @@ - + @@ -28548,7 +28526,7 @@ - + @@ -28557,7 +28535,7 @@ - + @@ -28625,12 +28603,12 @@ - + - + @@ -28647,19 +28625,19 @@ - + - + - + @@ -28694,12 +28672,12 @@ - + - + @@ -28712,44 +28690,44 @@ - + - + - + - + - + - + - + - + - + @@ -28758,32 +28736,32 @@ - + - + - + - + - + @@ -28808,7 +28786,7 @@ - + @@ -28816,12 +28794,12 @@ - - + + - + @@ -28831,33 +28809,33 @@ - + - + - + - + - + @@ -28865,85 +28843,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -28952,62 +28930,62 @@ - + - + - + - + - + - + - + - + - - - + + + - + - - + + - + - - - - - + + + + + @@ -29019,8 +28997,8 @@ - - + + @@ -29066,12 +29044,12 @@ - + - + @@ -29079,8 +29057,8 @@ - - + + @@ -29089,53 +29067,53 @@ - + - + - + - + - + - - - + + + - + - - + + - + - - - - - - + + + + + + @@ -29151,7 +29129,7 @@ - + @@ -29160,21 +29138,21 @@ - + - + - + - + @@ -29190,65 +29168,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29256,18 +29234,18 @@ - + - + - + - + @@ -29288,10 +29266,10 @@ - + - + @@ -29303,16 +29281,16 @@ - + - + - - - - + + + + @@ -29414,21 +29392,21 @@ - + - - - - + + + + - + - + @@ -29437,27 +29415,27 @@ - + - + - + - + - + @@ -29523,16 +29501,16 @@ - - + + - + - + @@ -29546,7 +29524,7 @@ - + @@ -29558,23 +29536,23 @@ - - + + - + - + - + @@ -29592,11 +29570,11 @@ - + - + @@ -29606,36 +29584,36 @@ - + - + - + - + - - - - + + + + - + - + - + - + @@ -29647,25 +29625,25 @@ - - + + - + - + - + - + - + - + @@ -29673,7 +29651,7 @@ - + @@ -29681,38 +29659,38 @@ - + - + - + - + - - - - - + + + + + - + - + - + @@ -29723,17 +29701,17 @@ - - - + + + - - - - - - - + + + + + + + @@ -29903,13 +29881,13 @@ - + - - + + @@ -29941,7 +29919,7 @@ - + @@ -30289,7 +30267,7 @@ - + @@ -30298,8 +30276,8 @@ - - + + @@ -30364,19 +30342,19 @@ - + - - + + - - - - + + + + @@ -30396,7 +30374,7 @@ - + @@ -30458,7 +30436,7 @@ - + @@ -30469,7 +30447,7 @@ - + @@ -30482,24 +30460,24 @@ - + - + - + - + - + @@ -30807,7 +30785,7 @@ - + @@ -30821,7 +30799,7 @@ - + @@ -30858,22 +30836,22 @@ - - - - + + + + - + - - + + - - + + @@ -31004,7 +30982,7 @@ - + @@ -31013,8 +30991,8 @@ - - + + @@ -31035,7 +31013,7 @@ - + @@ -31130,10 +31108,10 @@ - - - - + + + + @@ -31163,7 +31141,7 @@ - + @@ -31184,8 +31162,8 @@ - - + + @@ -31207,26 +31185,26 @@ - - - - - + + + + + + + + + + - - - - - - + - + - + @@ -31259,7 +31237,7 @@ - + @@ -31280,7 +31258,7 @@ - + @@ -31310,7 +31288,7 @@ - + @@ -31319,17 +31297,17 @@ - + - + - - + + @@ -31338,11 +31316,11 @@ - + - + @@ -31377,13 +31355,13 @@ - + - + @@ -31397,8 +31375,8 @@ - - + + diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 45fa47d62c78a3..1272927413868b 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -335,24 +335,9 @@ typedef struct _sharedexception { PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); -typedef struct xi_session _PyXI_session; -typedef struct _sharedns _PyXI_namespace; - -PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns); -PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names); -PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict( - _PyXI_namespace *ns, - PyObject *nsobj, - _PyXI_session *session); -PyAPI_FUNC(int) _PyXI_ApplyNamespace( - _PyXI_namespace *ns, - PyObject *nsobj, - PyObject *dflt); - - // A cross-interpreter session involves entering an interpreter -// (_PyXI_Enter()), doing some work with it, and finally exiting -// that interpreter (_PyXI_Exit()). +// with _PyXI_Enter(), doing some work with it, and finally exiting +// that interpreter with _PyXI_Exit(). // // At the boundaries of the session, both entering and exiting, // data may be exchanged between the previous interpreter and the @@ -360,39 +345,10 @@ PyAPI_FUNC(int) _PyXI_ApplyNamespace( // isolation between interpreters. This includes setting objects // in the target's __main__ module on the way in, and capturing // uncaught exceptions on the way out. -struct xi_session { - // Once a session has been entered, this is the tstate that was - // current before the session. If it is different from cur_tstate - // then we must have switched interpreters. Either way, this will - // be the current tstate once we exit the session. - PyThreadState *prev_tstate; - // Once a session has been entered, this is the current tstate. - // It must be current when the session exits. - PyThreadState *init_tstate; - // This is true if init_tstate needs cleanup during exit. - int own_init_tstate; - - // This is true if, while entering the session, init_thread took - // "ownership" of the interpreter's __main__ module. This means - // it is the only thread that is allowed to run code there. - // (Caveat: for now, users may still run exec() against the - // __main__ module's dict, though that isn't advisable.) - int running; - // This is a cached reference to the __dict__ of the entered - // interpreter's __main__ module. It is looked up when at the - // beginning of the session as a convenience. - PyObject *main_ns; - - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *error_override; - // This is set if exit captured an exception to propagate. - _PyXI_error *error; - - // -- pre-allocated memory -- - _PyXI_error _error; - _PyXI_errcode _error_override; -}; +typedef struct xi_session _PyXI_session; + +PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); +PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, @@ -400,6 +356,8 @@ PyAPI_FUNC(int) _PyXI_Enter( PyObject *nsupdates); PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace(_PyXI_session *); + PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 7f84b38a70e127..376517ab92360f 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -444,42 +444,54 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, PyObject **p_excinfo) { assert(!_PyErr_Occurred(tstate)); - _PyXI_session session = {0}; + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + return -1; + } // Prep and switch interpreters. - if (_PyXI_Enter(&session, interp, shareables) < 0) { + if (_PyXI_Enter(session, interp, shareables) < 0) { if (_PyErr_Occurred(tstate)) { // If an error occured at this step, it means that interp // was not prepared and switched. + _PyXI_FreeSession(session); return -1; } // Now, apply the error from another interpreter: - PyObject *excinfo = _PyXI_ApplyError(session.error); + PyObject *excinfo = _PyXI_ApplyCapturedException(session); if (excinfo != NULL) { *p_excinfo = excinfo; } assert(PyErr_Occurred()); + _PyXI_FreeSession(session); return -1; } // Run the script. - int res = _run_script(script, session.main_ns); + int res = -1; + PyObject *mainns = _PyXI_GetMainNamespace(session); + if (mainns == NULL) { + goto finally; + } + res = _run_script(script, mainns); +finally: // Clean up and switch back. - _PyXI_Exit(&session); + _PyXI_Exit(session); // Propagate any exception out to the caller. assert(!PyErr_Occurred()); if (res < 0) { - PyObject *excinfo = _PyXI_ApplyCapturedException(&session); + PyObject *excinfo = _PyXI_ApplyCapturedException(session); if (excinfo != NULL) { *p_excinfo = excinfo; } } else { - assert(!_PyXI_HasCapturedException(&session)); + assert(!_PyXI_HasCapturedException(session)); } + _PyXI_FreeSession(session); return res; } @@ -824,22 +836,27 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) } } - _PyXI_session session = {0}; + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + return NULL; + } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(&session, interp, updates) < 0) { + if (_PyXI_Enter(session, interp, updates) < 0) { if (!PyErr_Occurred()) { - _PyXI_ApplyCapturedException(&session); + _PyXI_ApplyCapturedException(session); assert(PyErr_Occurred()); } else { - assert(!_PyXI_HasCapturedException(&session)); + assert(!_PyXI_HasCapturedException(session)); } + _PyXI_FreeSession(session); return NULL; } // Clean up and switch back. - _PyXI_Exit(&session); + _PyXI_Exit(session); + _PyXI_FreeSession(session); Py_RETURN_NONE; } diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 6681b969183925..65ccab32daf730 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1914,156 +1914,212 @@ _sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) return res; } -struct _sharedns { - Py_ssize_t len; - _PyXI_namespace_item *items; -}; -static _PyXI_namespace * -_sharedns_new(void) -{ - _PyXI_namespace *ns = PyMem_RawCalloc(sizeof(_PyXI_namespace), 1); - if (ns == NULL) { - PyErr_NoMemory(); - return NULL; - } - *ns = (_PyXI_namespace){ 0 }; - return ns; -} +typedef struct { + Py_ssize_t maxitems; + Py_ssize_t numnames; + Py_ssize_t numvalues; + _PyXI_namespace_item items[1]; +} _PyXI_namespace; +#ifndef NDEBUG static int -_sharedns_is_initialized(_PyXI_namespace *ns) +_sharedns_check_counts(_PyXI_namespace *ns) { - if (ns->len == 0) { - assert(ns->items == NULL); + if (ns->maxitems <= 0) { + return 0; + } + if (ns->numnames < 0) { + return 0; + } + if (ns->numnames > ns->maxitems) { + return 0; + } + if (ns->numvalues < 0) { + return 0; + } + if (ns->numvalues > ns->numnames) { return 0; } - - assert(ns->len > 0); - assert(ns->items != NULL); - assert(_sharednsitem_is_initialized(&ns->items[0])); - assert(ns->len == 1 - || _sharednsitem_is_initialized(&ns->items[ns->len - 1])); return 1; } -#define HAS_COMPLETE_DATA 1 -#define HAS_PARTIAL_DATA 2 - static int -_sharedns_has_xidata(_PyXI_namespace *ns, int64_t *p_interpid) +_sharedns_check_consistency(_PyXI_namespace *ns) { - // We expect _PyXI_namespace to always be initialized. - assert(_sharedns_is_initialized(ns)); - int res = 0; - _PyXI_namespace_item *item0 = &ns->items[0]; - if (!_sharednsitem_is_initialized(item0)) { + if (!_sharedns_check_counts(ns)) { return 0; } - int64_t interpid0 = -1; - if (!_sharednsitem_has_value(item0, &interpid0)) { - return 0; + + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + item = &ns->items[0]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid0 = -1; + if (!_sharednsitem_has_value(item, &interpid0)) { + return 0; + } + i += 1; + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid = -1; + if (!_sharednsitem_has_value(item, &interpid)) { + return 0; + } + if (interpid != interpid0) { + return 0; + } + } } - if (ns->len > 1) { - // At this point we know it is has at least partial data. - _PyXI_namespace_item *itemN = &ns->items[ns->len-1]; - if (!_sharednsitem_is_initialized(itemN)) { - res = HAS_PARTIAL_DATA; - goto finally; + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; } - int64_t interpidN = -1; - if (!_sharednsitem_has_value(itemN, &interpidN)) { - res = HAS_PARTIAL_DATA; - goto finally; + if (_sharednsitem_has_value(item, NULL)) { + return 0; } - assert(interpidN == interpid0); } - res = HAS_COMPLETE_DATA; - *p_interpid = interpid0; - -finally: - return res; + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + if (_sharednsitem_is_initialized(item)) { + return 0; + } + if (_sharednsitem_has_value(item, NULL)) { + return 0; + } + } + return 1; } +#endif -static void -_sharedns_clear(_PyXI_namespace *ns) +static _PyXI_namespace * +_sharedns_alloc(Py_ssize_t maxitems) { - if (!_sharedns_is_initialized(ns)) { - return; + if (maxitems < 0) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return NULL; + } + else if (maxitems == 0) { + PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); + return NULL; } - // If the cross-interpreter data were allocated as part of - // _PyXI_namespace_item (instead of dynamically), this is where - // we would need verify that we are clearing the items in the - // correct interpreter, to avoid a race with releasing the XI data - // via a pending call. See _sharedns_has_xidata(). - for (Py_ssize_t i=0; i < ns->len; i++) { - _sharednsitem_clear(&ns->items[i]); + // Check for overflow. + size_t fixedsize = sizeof(_PyXI_namespace) - sizeof(_PyXI_namespace_item); + if ((size_t)maxitems > + ((size_t)PY_SSIZE_T_MAX - fixedsize) / sizeof(_PyXI_namespace_item)) + { + PyErr_NoMemory(); + return NULL; + } + + // Allocate the value, including items. + size_t size = fixedsize + sizeof(_PyXI_namespace_item) * maxitems; + + _PyXI_namespace *ns = PyMem_RawCalloc(size, 1); + if (ns == NULL) { + PyErr_NoMemory(); + return NULL; } - PyMem_RawFree(ns->items); - ns->items = NULL; - ns->len = 0; + ns->maxitems = maxitems; + assert(_sharedns_check_consistency(ns)); + return ns; } static void _sharedns_free(_PyXI_namespace *ns) { - _sharedns_clear(ns); + // If we weren't always dynamically allocating the cross-interpreter + // data in each item then we would need to use a pending call + // to call _sharedns_free(), to avoid the race between freeing + // the shared namespace and releasing the XI data. + assert(_sharedns_check_counts(ns)); + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + // One or more items may have interpreter-specific data. +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + int64_t interpid_i; +#endif + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + // While we do want to ensure consistency across items, + // technically they don't need to match the current + // interpreter. However, we keep the constraint for + // simplicity, by giving _PyXI_FreeNamespace() the exclusive + // responsibility of dealing with the owning interpreter. + assert(_sharednsitem_has_value(item, &interpid_i)); + assert(interpid_i == interpid); + _sharednsitem_clear(item); + } + } + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + _sharednsitem_clear(item); + } +#ifndef NDEBUG + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + assert(!_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + } +#endif + PyMem_RawFree(ns); } -static int -_sharedns_init(_PyXI_namespace *ns, PyObject *names) +static _PyXI_namespace * +_create_sharedns(PyObject *names) { - assert(!_sharedns_is_initialized(ns)); assert(names != NULL); - Py_ssize_t len = PyDict_CheckExact(names) + Py_ssize_t numnames = PyDict_CheckExact(names) ? PyDict_Size(names) : PySequence_Size(names); - if (len < 0) { - return -1; - } - if (len == 0) { - PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); - return -1; - } - assert(len > 0); - // Allocate the items. - _PyXI_namespace_item *items = - PyMem_RawCalloc(sizeof(struct _sharednsitem), len); - if (items == NULL) { - PyErr_NoMemory(); - return -1; + _PyXI_namespace *ns = _sharedns_alloc(numnames); + if (ns == NULL) { + return NULL; } + _PyXI_namespace_item *items = ns->items; // Fill in the names. - Py_ssize_t i = -1; if (PyDict_CheckExact(names)) { + Py_ssize_t i = 0; Py_ssize_t pos = 0; - for (i=0; i < len; i++) { - PyObject *key; - if (!PyDict_Next(names, &pos, &key, NULL)) { - // This should not be possible. - assert(0); - goto error; - } - if (_sharednsitem_init(&items[i], key) < 0) { + PyObject *name; + while(PyDict_Next(names, &pos, &name, NULL)) { + if (_sharednsitem_init(&items[i], name) < 0) { goto error; } + ns->numnames += 1; + i += 1; } } else if (PySequence_Check(names)) { - for (i=0; i < len; i++) { - PyObject *key = PySequence_GetItem(names, i); - if (key == NULL) { + for (Py_ssize_t i = 0; i < numnames; i++) { + PyObject *name = PySequence_GetItem(names, i); + if (name == NULL) { goto error; } - int res = _sharednsitem_init(&items[i], key); - Py_DECREF(key); + int res = _sharednsitem_init(&items[i], name); + Py_DECREF(name); if (res < 0) { goto error; } + ns->numnames += 1; } } else { @@ -2071,140 +2127,79 @@ _sharedns_init(_PyXI_namespace *ns, PyObject *names) "non-sequence namespace not supported"); goto error; } - - ns->items = items; - ns->len = len; - assert(_sharedns_is_initialized(ns)); - return 0; + assert(ns->numnames == ns->maxitems); + return ns; error: - for (Py_ssize_t j=0; j < i; j++) { - _sharednsitem_clear(&items[j]); - } - PyMem_RawFree(items); - assert(!_sharedns_is_initialized(ns)); - return -1; -} - -void -_PyXI_FreeNamespace(_PyXI_namespace *ns) -{ - if (!_sharedns_is_initialized(ns)) { - return; - } - - int64_t interpid = -1; - if (!_sharedns_has_xidata(ns, &interpid)) { - _sharedns_free(ns); - return; - } - - if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) { - _sharedns_free(ns); - } - else { - // If we weren't always dynamically allocating the cross-interpreter - // data in each item then we would need to using a pending call - // to call _sharedns_free(), to avoid the race between freeing - // the shared namespace and releasing the XI data. - _sharedns_free(ns); - } -} - -_PyXI_namespace * -_PyXI_NamespaceFromNames(PyObject *names) -{ - if (names == NULL || names == Py_None) { - return NULL; - } - - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; - } - - if (_sharedns_init(ns, names) < 0) { - PyMem_RawFree(ns); - if (PySequence_Size(names) == 0) { - PyErr_Clear(); - } - return NULL; - } - - return ns; + _sharedns_free(ns); + return NULL; } -#ifndef NDEBUG -static int _session_is_active(_PyXI_session *); -#endif static void _propagate_not_shareable_error(_PyXI_session *); -int -_PyXI_FillNamespaceFromDict(_PyXI_namespace *ns, PyObject *nsobj, - _PyXI_session *session) -{ - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - assert(_sharedns_is_initialized(ns)); - for (Py_ssize_t i=0; i < ns->len; i++) { - _PyXI_namespace_item *item = &ns->items[i]; - if (_sharednsitem_copy_from_ns(item, nsobj) < 0) { +static int +_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, _PyXI_session *session) +{ + // All items are expected to be shareable. + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + assert(ns->numvalues == 0); + for (Py_ssize_t i=0; i < ns->maxitems; i++) { + if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj) < 0) { _propagate_not_shareable_error(session); // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { _sharednsitem_clear_value(&ns->items[j]); + ns->numvalues -= 1; } return -1; } + ns->numvalues += 1; } return 0; } -// All items are expected to be shareable. -static _PyXI_namespace * -_PyXI_NamespaceFromDict(PyObject *nsobj, _PyXI_session *session) +static int +_sharedns_free_pending(void *data) { - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - if (nsobj == NULL || nsobj == Py_None) { - return NULL; - } - if (!PyDict_CheckExact(nsobj)) { - PyErr_SetString(PyExc_TypeError, "expected a dict"); - return NULL; - } + _sharedns_free((_PyXI_namespace *)data); + return 0; +} - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; +static void +_destroy_sharedns(_PyXI_namespace *ns) +{ + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + if (ns->numvalues == 0) { + _sharedns_free(ns); + return; } - if (_sharedns_init(ns, nsobj) < 0) { - if (PyDict_Size(nsobj) == 0) { - PyMem_RawFree(ns); - PyErr_Clear(); - return NULL; - } - goto error; + int64_t interpid0; + if (!_sharednsitem_has_value(&ns->items[0], &interpid0)) { + // This shouldn't have been possible. + // We can deal with it in _sharedns_free(). + _sharedns_free(ns); + return; } - - if (_PyXI_FillNamespaceFromDict(ns, nsobj, session) < 0) { - goto error; + PyInterpreterState *interp = _PyInterpreterState_LookUpID(interpid0); + if (interp == PyInterpreterState_Get()) { + _sharedns_free(ns); + return; } - return ns; - -error: - assert(PyErr_Occurred() - || (session != NULL && session->error_override != NULL)); - _sharedns_free(ns); - return NULL; + // One or more items may have interpreter-specific data. + // Currently the xidata for each value is dynamically allocated, + // so technically we don't need to worry about that. + // However, explicitly adding a pending call here is simpler. + (void)_Py_CallInInterpreter(interp, _sharedns_free_pending, ns); } -int -_PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) +static int +_apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) { - for (Py_ssize_t i=0; i < ns->len; i++) { + for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_apply(&ns->items[i], nsobj, dflt) != 0) { return -1; } @@ -2213,9 +2208,79 @@ _PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) } -/**********************/ -/* high-level helpers */ -/**********************/ +/*********************************/ +/* switched-interpreter sessions */ +/*********************************/ + +struct xi_session { +#define SESSION_UNUSED 0 +#define SESSION_ACTIVE 1 + int status; + int switched; + + // Once a session has been entered, this is the tstate that was + // current before the session. If it is different from cur_tstate + // then we must have switched interpreters. Either way, this will + // be the current tstate once we exit the session. + PyThreadState *prev_tstate; + // Once a session has been entered, this is the current tstate. + // It must be current when the session exits. + PyThreadState *init_tstate; + // This is true if init_tstate needs cleanup during exit. + int own_init_tstate; + + // This is true if, while entering the session, init_thread took + // "ownership" of the interpreter's __main__ module. This means + // it is the only thread that is allowed to run code there. + // (Caveat: for now, users may still run exec() against the + // __main__ module's dict, though that isn't advisable.) + int running; + // This is a cached reference to the __dict__ of the entered + // interpreter's __main__ module. It is looked up when at the + // beginning of the session as a convenience. + PyObject *main_ns; + + // This is set if the interpreter is entered and raised an exception + // that needs to be handled in some special way during exit. + _PyXI_errcode *error_override; + // This is set if exit captured an exception to propagate. + _PyXI_error *error; + + // -- pre-allocated memory -- + _PyXI_error _error; + _PyXI_errcode _error_override; +}; + + +_PyXI_session * +_PyXI_NewSession(void) +{ + _PyXI_session *session = PyMem_RawCalloc(1, sizeof(_PyXI_session)); + if (session == NULL) { + PyErr_NoMemory(); + return NULL; + } + return session; +} + +void +_PyXI_FreeSession(_PyXI_session *session) +{ + assert(session->status == SESSION_UNUSED); + PyMem_RawFree(session); +} + + +static inline int +_session_is_active(_PyXI_session *session) +{ + return session->status == SESSION_ACTIVE; +} + +static int _ensure_main_ns(_PyXI_session *); +static inline void _session_set_error(_PyXI_session *, _PyXI_errcode); +static void _capture_current_exception(_PyXI_session *); + /* enter/exit a cross-interpreter session */ @@ -2223,6 +2288,7 @@ static void _enter_session(_PyXI_session *session, PyInterpreterState *interp) { // Set here and cleared in _exit_session(). + assert(session->status == SESSION_UNUSED); assert(!session->own_init_tstate); assert(session->init_tstate == NULL); assert(session->prev_tstate == NULL); @@ -2237,15 +2303,22 @@ _enter_session(_PyXI_session *session, PyInterpreterState *interp) // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); PyThreadState *prev = tstate; - if (interp != tstate->interp) { + int same_interp = (interp == tstate->interp); + if (!same_interp) { tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); // XXX Possible GILState issues? - session->prev_tstate = PyThreadState_Swap(tstate); - assert(session->prev_tstate == prev); - session->own_init_tstate = 1; + PyThreadState *swapped = PyThreadState_Swap(tstate); + assert(swapped == prev); + (void)swapped; } - session->init_tstate = tstate; - session->prev_tstate = prev; + + *session = (_PyXI_session){ + .status = SESSION_ACTIVE, + .switched = !same_interp, + .init_tstate = tstate, + .prev_tstate = prev, + .own_init_tstate = !same_interp, + }; } static void @@ -2256,9 +2329,7 @@ _exit_session(_PyXI_session *session) assert(PyThreadState_Get() == tstate); // Release any of the entered interpreters resources. - if (session->main_ns != NULL) { - Py_CLEAR(session->main_ns); - } + Py_CLEAR(session->main_ns); // Ensure this thread no longer owns __main__. if (session->running) { @@ -2279,17 +2350,15 @@ _exit_session(_PyXI_session *session) else { assert(!session->own_init_tstate); } - session->prev_tstate = NULL; - session->init_tstate = NULL; -} -#ifndef NDEBUG -static int -_session_is_active(_PyXI_session *session) -{ - return (session->init_tstate != NULL); + // For now the error data persists past the exit. + *session = (_PyXI_session){ + .error_override = session->error_override, + .error = session->error, + ._error = session->_error, + ._error_override = session->_error_override, + }; } -#endif static void _propagate_not_shareable_error(_PyXI_session *session) @@ -2306,11 +2375,102 @@ _propagate_not_shareable_error(_PyXI_session *session) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - session->_error_override = _PyXI_ERR_NOT_SHAREABLE; - session->error_override = &session->_error_override; + _session_set_error(session, _PyXI_ERR_NOT_SHAREABLE); } } +PyObject * +_PyXI_ApplyCapturedException(_PyXI_session *session) +{ + assert(!PyErr_Occurred()); + assert(session->error != NULL); + PyObject *res = _PyXI_ApplyError(session->error); + assert((res == NULL) != (PyErr_Occurred() == NULL)); + session->error = NULL; + return res; +} + +int +_PyXI_HasCapturedException(_PyXI_session *session) +{ + return session->error != NULL; +} + +int +_PyXI_Enter(_PyXI_session *session, + PyInterpreterState *interp, PyObject *nsupdates) +{ + // Convert the attrs for cross-interpreter use. + _PyXI_namespace *sharedns = NULL; + if (nsupdates != NULL) { + Py_ssize_t len = PyDict_Size(nsupdates); + if (len < 0) { + return -1; + } + if (len > 0) { + sharedns = _create_sharedns(nsupdates); + if (sharedns == NULL) { + return -1; + } + if (_fill_sharedns(sharedns, nsupdates, NULL) < 0) { + assert(session->error == NULL); + _destroy_sharedns(sharedns); + return -1; + } + } + } + + // Switch to the requested interpreter (if necessary). + _enter_session(session, interp); + _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + + // Ensure this thread owns __main__. + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + // In the case where we didn't switch interpreters, it would + // be more efficient to leave the exception in place and return + // immediately. However, life is simpler if we don't. + errcode = _PyXI_ERR_ALREADY_RUNNING; + goto error; + } + session->running = 1; + + // Apply the cross-interpreter data. + if (sharedns != NULL) { + if (_ensure_main_ns(session) < 0) { + errcode = _PyXI_ERR_MAIN_NS_FAILURE; + goto error; + } + if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { + errcode = _PyXI_ERR_APPLY_NS_FAILURE; + goto error; + } + _destroy_sharedns(sharedns); + } + + errcode = _PyXI_ERR_NO_ERROR; + assert(!PyErr_Occurred()); + return 0; + +error: + // We want to propagate all exceptions here directly (best effort). + _session_set_error(session, errcode); + _exit_session(session); + if (sharedns != NULL) { + _destroy_sharedns(sharedns); + } + return -1; +} + +void +_PyXI_Exit(_PyXI_session *session) +{ + _capture_current_exception(session); + _exit_session(session); +} + + +/* in an active cross-interpreter session */ + static void _capture_current_exception(_PyXI_session *session) { @@ -2372,100 +2532,55 @@ _capture_current_exception(_PyXI_session *session) // Finished! assert(!PyErr_Occurred()); - session->error = err; -} - -PyObject * -_PyXI_ApplyCapturedException(_PyXI_session *session) -{ - assert(!PyErr_Occurred()); - assert(session->error != NULL); - PyObject *res = _PyXI_ApplyError(session->error); - assert((res == NULL) != (PyErr_Occurred() == NULL)); - session->error = NULL; - return res; + session->error = err; } -int -_PyXI_HasCapturedException(_PyXI_session *session) +static inline void +_session_set_error(_PyXI_session *session, _PyXI_errcode errcode) { - return session->error != NULL; + assert(_session_is_active(session)); + assert(PyErr_Occurred()); + if (errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION) { + session->_error_override = errcode; + session->error_override = &session->_error_override; + } + _capture_current_exception(session); } -int -_PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates) +static int +_ensure_main_ns(_PyXI_session *session) { - // Convert the attrs for cross-interpreter use. - _PyXI_namespace *sharedns = NULL; - if (nsupdates != NULL) { - sharedns = _PyXI_NamespaceFromDict(nsupdates, NULL); - if (sharedns == NULL && PyErr_Occurred()) { - assert(session->error == NULL); - return -1; - } - } - - // Switch to the requested interpreter (if necessary). - _enter_session(session, interp); - PyThreadState *session_tstate = session->init_tstate; - _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; - - // Ensure this thread owns __main__. - if (_PyInterpreterState_SetRunningMain(interp) < 0) { - // In the case where we didn't switch interpreters, it would - // be more efficient to leave the exception in place and return - // immediately. However, life is simpler if we don't. - errcode = _PyXI_ERR_ALREADY_RUNNING; - goto error; + assert(_session_is_active(session)); + if (session->main_ns != NULL) { + return 0; } - session->running = 1; - // Cache __main__.__dict__. - PyObject *main_mod = _Py_GetMainModule(session_tstate); + PyObject *main_mod = _Py_GetMainModule(session->init_tstate); if (_Py_CheckMainModule(main_mod) < 0) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + return -1; } session->main_ns = Py_NewRef(ns); - - // Apply the cross-interpreter data. - if (sharedns != NULL) { - if (_PyXI_ApplyNamespace(sharedns, ns, NULL) < 0) { - errcode = _PyXI_ERR_APPLY_NS_FAILURE; - goto error; - } - _PyXI_FreeNamespace(sharedns); - } - - errcode = _PyXI_ERR_NO_ERROR; - assert(!PyErr_Occurred()); return 0; - -error: - assert(PyErr_Occurred()); - // We want to propagate all exceptions here directly (best effort). - assert(errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION); - session->error_override = &errcode; - _capture_current_exception(session); - _exit_session(session); - if (sharedns != NULL) { - _PyXI_FreeNamespace(sharedns); - } - return -1; } -void -_PyXI_Exit(_PyXI_session *session) +PyObject * +_PyXI_GetMainNamespace(_PyXI_session *session) { - _capture_current_exception(session); - _exit_session(session); + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return NULL; + } + if (_ensure_main_ns(session) < 0) { + _session_set_error(session, _PyXI_ERR_MAIN_NS_FAILURE); + _capture_current_exception(session); + return NULL; + } + return session->main_ns; } From 7480e50217299cf0bf69abce8f3e9d0ef1fcdef9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 19:49:05 +0200 Subject: [PATCH 206/713] [3.14] gh-132775: Fix Recently Introduced Warnings (gh-134532) (cherry picked from commit ac06b534eed, AKA gh-134530) Co-authored-by: Eric Snow --- Modules/_interpchannelsmodule.c | 2 +- Modules/_interpqueuesmodule.c | 2 +- Python/crossinterp.c | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 0ab553190001bd..bfd805bf5e4072 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -3443,7 +3443,7 @@ channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t cid = cid_data.cid; - struct _channeldefaults defaults; + struct _channeldefaults defaults = {0}; int err = channel_get_defaults(&_globals.channels, cid, &defaults); if (handle_channel_error(err, self, cid)) { return NULL; diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 816285c9eff44a..ffc52c8ee74d85 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -1772,7 +1772,7 @@ queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t qid = qidarg.id; - struct _queuedefaults defaults; + struct _queuedefaults defaults = {0}; int err = queue_get_defaults(&_globals.queues, qid, &defaults); if (handle_queue_error(err, self, qid)) { return NULL; diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 65ccab32daf730..26eecdddf4bdd0 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -1795,6 +1795,7 @@ typedef struct _sharednsitem { // in a different interpreter to release the XI data. } _PyXI_namespace_item; +#ifndef NDEBUG static int _sharednsitem_is_initialized(_PyXI_namespace_item *item) { @@ -1803,6 +1804,7 @@ _sharednsitem_is_initialized(_PyXI_namespace_item *item) } return 0; } +#endif static int _sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) From 37c2c8c9d8a20fdc9e9d5247d96ea6719de3134d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 20:32:49 +0200 Subject: [PATCH 207/713] [3.14] Docs: Add note to tutorial clarifying scope (GH-134534) (#134536) Docs: Add note to tutorial clarifying scope (GH-134534) * Add note to tutorial * Update formatting (cherry picked from commit 742d5b5c5d75eae44c66a43ebfa24a4f286ea8a1) Co-authored-by: Kattni --- Doc/tutorial/index.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index 96791f88c867ab..d0bf77dc40d0a1 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -4,6 +4,10 @@ The Python Tutorial ###################### +.. Tip:: This tutorial is designed for + *programmers* that are new to the Python language, + **not** *beginners* who are new to programming. + Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, @@ -21,7 +25,8 @@ implemented in C or C++ (or other languages callable from C). Python is also suitable as an extension language for customizable applications. This tutorial introduces the reader informally to the basic concepts and -features of the Python language and system. It helps to have a Python +features of the Python language and system. Be aware that it expects you to +have a basic understanding of programming in general. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. From 06a3a85f15d482b5a6880925417bc66f481efedc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 23:37:52 +0200 Subject: [PATCH 208/713] [3.14] Avoid `__file__` in `hashlib` example (GH-134540) (#134549) Avoid `__file__` in `hashlib` example (GH-134540) (cherry picked from commit 9b292ff0223dab0f999d5c1e13355552fd7cd528) Co-authored-by: Josh Cannon --- Doc/library/hashlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index bb2d2fad23bdb8..4818a4944a512a 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -284,7 +284,7 @@ a file or file-like object. Example: >>> import io, hashlib, hmac - >>> with open(hashlib.__file__, "rb") as f: + >>> with open("library/hashlib.rst", "rb") as f: ... digest = hashlib.file_digest(f, "sha256") ... >>> digest.hexdigest() # doctest: +ELLIPSIS From 15e26eebf79d2337eb20338fd9c90f7d3bcd047c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 22 May 2025 23:42:23 +0200 Subject: [PATCH 209/713] [3.14] GH-130397: remove special-casing of C stack depth for WASI (GH-134469) (GH-134547) GH-130397: remove special-casing of C stack depth for WASI (GH-134469) Removed special-casing for WASI when setting C stack depth limits. Since WASI has its own C stack checking this isn't a security risk. Also disabled some tests that stopped passing. They all happened to have already been disabled under Emscripten. (cherry picked from commit ad42dc1909bdf8ec775b63fb22ed48ff42797a17) Co-authored-by: Brett Cannon --- Include/pythonrun.h | 3 --- Lib/test/test_copy.py | 3 +++ Lib/test/test_descr.py | 2 ++ Lib/test/test_exception_group.py | 4 +++- Lib/test/test_isinstance.py | 2 ++ Lib/test/test_json/test_recursion.py | 2 ++ .../2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst | 3 +++ Python/ceval.c | 3 --- 8 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst diff --git a/Include/pythonrun.h b/Include/pythonrun.h index fad2b3c77476e4..716c4ab3c81cae 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -29,9 +29,6 @@ PyAPI_DATA(int) (*PyOS_InputHook)(void); # define PYOS_LOG2_STACK_MARGIN 12 #elif defined(Py_DEBUG) && defined(WIN32) # define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really a size */ -# define PYOS_LOG2_STACK_MARGIN 9 #else # define PYOS_LOG2_STACK_MARGIN 11 #endif diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 3cb8072abefe98..467ec09d99e462 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -372,6 +372,7 @@ def test_deepcopy_list(self): self.assertIsNot(x[0], y[0]) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -400,6 +401,7 @@ def test_deepcopy_tuple_of_immutables(self): self.assertIs(x, y) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -418,6 +420,7 @@ def test_deepcopy_dict(self): self.assertIsNot(x["foo"], y["foo"]) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d635855f537af0..ea076ba4fef2db 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3943,6 +3943,7 @@ def __del__(self): del C.__del__ @unittest.skipIf(support.is_emscripten, "Seems to works in Pyodide?") + @support.skip_wasi_stack_overflow() def test_slots_trash(self): # Testing slot trash... # Deallocating deeply nested slotted trash caused stack overflows @@ -4868,6 +4869,7 @@ class Thing: deque.append(thing, thing) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 242d7ced007b10..5df2c41c6b56bc 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import skip_emscripten_stack_overflow, exceeds_recursion_limit +from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -465,12 +465,14 @@ def make_deep_eg(self): return e @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index daad00e86432d0..f440fc28ee7b7d 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -318,6 +318,7 @@ def __bases__(self): self.assertRaises(RecursionError, isinstance, 1, X()) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): @@ -328,6 +329,7 @@ def __getattr__(self, attr): issubclass(Failure(), int) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_cycle_in_bases(self): """Regression test for bpo-30570.""" class X: diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index d82093f3895167..8f0e5e078ed0d4 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -69,6 +69,7 @@ def default(self, o): @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): very_deep = 200000 # test that loading highly-nested objects doesn't segfault when C @@ -98,6 +99,7 @@ def test_highly_nested_objects_encoding(self): self.dumps(d) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst new file mode 100644 index 00000000000000..34a2f4d1278e74 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst @@ -0,0 +1,3 @@ +Remove special-casing for C stack depth limits for WASI. Due to +WebAssembly's built-in stack protection this does not pose a security +concern. diff --git a/Python/ceval.c b/Python/ceval.c index 490b653f132a6a..d1b6b9f1a1a35f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -360,9 +360,6 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) # define Py_C_STACK_SIZE 1200000 #elif defined(__sparc__) # define Py_C_STACK_SIZE 1600000 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really the stack depth */ -# define Py_C_STACK_SIZE 131072 // wasi-libc DEFAULT_STACK_SIZE #elif defined(__hppa__) || defined(__powerpc64__) # define Py_C_STACK_SIZE 2000000 #else From 8f4bf1538d2d05d502417b0da6623727519ab945 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 22 May 2025 15:07:21 -0700 Subject: [PATCH 210/713] [3.14] GH-131769: fix detecting a pydebug build of the build Python when building for WASI (GH-133219) --- ...-04-30-10-22-08.gh-issue-131769.H0oy5x.rst | 1 + Tools/wasm/wasi.py | 22 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Build/2025-04-30-10-22-08.gh-issue-131769.H0oy5x.rst diff --git a/Misc/NEWS.d/next/Build/2025-04-30-10-22-08.gh-issue-131769.H0oy5x.rst b/Misc/NEWS.d/next/Build/2025-04-30-10-22-08.gh-issue-131769.H0oy5x.rst new file mode 100644 index 00000000000000..834b0d9f7e145d --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-04-30-10-22-08.gh-issue-131769.H0oy5x.rst @@ -0,0 +1 @@ +Fix detecting when the build Python in a cross-build is a pydebug build. diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a742043e4be1d2..5f60f0ec469dd1 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -111,7 +111,7 @@ def call(command, *, quiet, **kwargs): def build_platform(): """The name of the build/host platform.""" - # Can also be found via `config.guess`.` + # Can also be found via `config.guess`. return sysconfig.get_config_var("BUILD_GNU_TYPE") @@ -127,6 +127,15 @@ def build_python_path(): return binary +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run([build_python_path(), "-c", test], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return bool(result.returncode) + + @subdir(BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" @@ -213,18 +222,15 @@ def configure_wasi_python(context, working_dir): lib_dirs = list(python_build_dir.glob("lib.*")) assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - if pydebug: - sysconfig_data += "-pydebug" + python_version = lib_dir.rpartition("-")[-1] + sysconfig_data_dir = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" # Use PYTHONPATH to include sysconfig data which must be anchored to the # WASI guest's `/` directory. args = {"GUEST_DIR": "/", "HOST_DIR": CHECKOUT, "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", + "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", "PYTHON_WASM": working_dir / "python.wasm"} # Check dynamically for wasmtime in case it was specified manually via # `--host-runner`. @@ -244,7 +250,7 @@ def configure_wasi_python(context, working_dir): f"--host={context.host_triple}", f"--build={build_platform()}", f"--with-build-python={build_python}"] - if pydebug: + if build_python_is_pydebug(): configure.append("--with-pydebug") if context.args: configure.extend(context.args) From ee3a0efc9decfceaf00448beacdb2af688c68744 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 02:42:48 +0200 Subject: [PATCH 211/713] [3.14] Docs: fix link in free-threading-python.rst (GH-134548) (#134555) --- Doc/howto/free-threading-python.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index f7a894ac2cd78e..c33cef2c8e935b 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -32,7 +32,7 @@ optionally support installing free-threaded Python binaries. The installers are available at https://www.python.org/downloads/. For information on other platforms, see the `Installing a Free-Threaded Python -`_, a +`_, a community-maintained installation guide for installing free-threaded Python. When building CPython from source, the :option:`--disable-gil` configure option From 1cdbb33771f737968f73b4e8bdce04551db1a417 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 06:06:19 +0200 Subject: [PATCH 212/713] [3.14] gh-128307: Update docs for asyncio.create_task, TaskGroup.create_task, asyncio.create_task (GH-134202) (#134553) gh-128307: Update docs for asyncio.create_task, TaskGroup.create_task, asyncio.create_task (GH-134202) (cherry picked from commit a3d0306ca08d09b1417d72df62e3b9d775143031) Co-authored-by: Thomas Grainger --- Doc/library/asyncio-eventloop.rst | 25 +++++++++++++++++++++++-- Doc/library/asyncio-task.rst | 21 +++++++++++++++++---- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 21f7d0547af1dd..91970c282391f7 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -361,7 +361,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None) +.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Schedule the execution of :ref:`coroutine ` *coro*. Return a :class:`Task` object. @@ -370,6 +370,10 @@ Creating Futures and Tasks for interoperability. In this case, the result type is a subclass of :class:`Task`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. + If the *name* argument is provided and not ``None``, it is set as the name of the task using :meth:`Task.set_name`. @@ -388,8 +392,15 @@ Creating Futures and Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.13.3 + Added ``kwargs`` which passes on arbitrary extra parameters, including ``name`` and ``context``. + + .. versionchanged:: 3.13.4 + Rolled back the change that passes on *name* and *context* (if it is None), + while still passing on other arbitrary keyword arguments (to avoid breaking backwards compatibility with 3.13.3). + .. versionchanged:: 3.14 - Added the *eager_start* parameter. + All *kwargs* are now passed on. The *eager_start* parameter works with eager task factories. .. method:: loop.set_task_factory(factory) @@ -402,6 +413,16 @@ Creating Futures and Tasks event loop, and *coro* is a coroutine object. The callable must pass on all *kwargs*, and return a :class:`asyncio.Task`-compatible object. + .. versionchanged:: 3.13.3 + Required that all *kwargs* are passed on to :class:`asyncio.Task`. + + .. versionchanged:: 3.13.4 + *name* is no longer passed to task factories. *context* is no longer passed + to task factories if it is ``None``. + + .. versionchanged:: 3.14 + *name* and *context* are now unconditionally passed on to task factories again. + .. method:: loop.get_task_factory() Return a task factory or ``None`` if the default one is in use. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 59acce1990ae04..b19ffa8213a971 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -238,18 +238,24 @@ Creating Tasks ----------------------------------------------- -.. function:: create_task(coro, *, name=None, context=None) +.. function:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. - If *name* is not ``None``, it is set as the name of the task using - :meth:`Task.set_name`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *coro* to run in. The current context copy is created when no *context* is provided. + An optional keyword-only *eager_start* argument allows specifying + if the task should execute eagerly during the call to create_task, + or be scheduled later. If *eager_start* is not passed the mode set + by :meth:`loop.set_task_factory` will be used. + The task is executed in the loop returned by :func:`get_running_loop`, :exc:`RuntimeError` is raised if there is no running loop in current thread. @@ -290,6 +296,9 @@ Creating Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.14 + Added the *eager_start* parameter by passing on all *kwargs*. + Task Cancellation ================= @@ -330,7 +339,7 @@ and reliable way to wait for all tasks in the group to finish. .. versionadded:: 3.11 - .. method:: create_task(coro, *, name=None, context=None) + .. method:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. @@ -342,6 +351,10 @@ and reliable way to wait for all tasks in the group to finish. Close the given coroutine if the task group is not active. + .. versionchanged:: 3.14 + + Passes on all *kwargs* to :meth:`loop.create_task` + Example:: async def main(): From c67eb41a3be8195e411da305246d7915a3ba960e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 06:58:51 +0200 Subject: [PATCH 213/713] [3.14] gh-114177: avoid calling connection lost callbacks when loop is already closed in asyncio subprocess (GH-134508) (#134561) gh-114177: avoid calling connection lost callbacks when loop is already closed in asyncio subprocess (GH-134508) (cherry picked from commit 5804ee7b467d86131be3ff7d569443efb0d0f9fd) Co-authored-by: Kumar Aditya --- Lib/asyncio/base_subprocess.py | 7 ++++++- .../Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst | 1 + 2 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 9c2ba679ce2bf1..d40af422e614c1 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -104,7 +104,12 @@ def close(self): for proto in self._pipes.values(): if proto is None: continue - proto.pipe.close() + # See gh-114177 + # skip closing the pipe if loop is already closed + # this can happen e.g. when loop is closed immediately after + # process is killed + if self._loop and not self._loop.is_closed(): + proto.pipe.close() if (self._proc is not None and # has the child process finished? diff --git a/Misc/NEWS.d/next/Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst b/Misc/NEWS.d/next/Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst new file mode 100644 index 00000000000000..c98fde5fb04f5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst @@ -0,0 +1 @@ +Fix :mod:`asyncio` to not close subprocess pipes which would otherwise error out when the event loop is already closed. From 73967c4c0149f04f3b5043a260f1a45ed23331b9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 07:40:56 +0200 Subject: [PATCH 214/713] [3.14] gh-134451: Converted `asyncio.tools.CycleFoundException` from dataclass to a regular exception type. (GH-134513) (#134564) gh-134451: Converted `asyncio.tools.CycleFoundException` from dataclass to a regular exception type. (GH-134513) (cherry picked from commit f9324cb3cb4d9bb9f0aef2e48b8afa895bde4b0d) Co-authored-by: Evgeny Demchenko --- Lib/asyncio/tools.py | 12 +++++++++--- .../2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index bf1cb5e64cbfbe..b2da7d2f6ba10c 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -13,11 +13,17 @@ class NodeType(Enum): TASK = 2 -@dataclass(frozen=True) class CycleFoundException(Exception): """Raised when there is a cycle when drawing the call tree.""" - cycles: list[list[int]] - id2name: dict[int, str] + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name + # ─── indexing helpers ─────────────────────────────────────────── diff --git a/Misc/NEWS.d/next/Library/2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst b/Misc/NEWS.d/next/Library/2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst new file mode 100644 index 00000000000000..3c8339f8842254 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst @@ -0,0 +1 @@ +Converted ``asyncio.tools.CycleFoundException`` from dataclass to a regular exception type. From 7efe67ae3597b654c7d0005763e148a5d0e9d9ef Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 11:51:17 +0200 Subject: [PATCH 215/713] [3.14] gh-106318: Add example for `str.center()` (GH-134518) (#134571) Co-authored-by: Blaise Pabon --- Doc/library/stdtypes.rst | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 3486a18b5cb1f0..ce613476854110 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1788,8 +1788,14 @@ expression support in the :mod:`re` module). Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. - + returned if *width* is less than or equal to ``len(s)``. For example:: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' .. method:: str.count(sub[, start[, end]]) From 896b745a7a191c144c27893933b8bf57b4fc77a9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 11:53:16 +0200 Subject: [PATCH 216/713] [3.14] gh-106318: Add example for `str.count()` (GH-134519) (#134574) Co-authored-by: Blaise Pabon --- Doc/library/stdtypes.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ce613476854110..31d71031bca12c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1805,8 +1805,18 @@ expression support in the :mod:`re` module). interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between characters - which is the length of the string plus one. - + which is the length of the string plus one. For example:: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 .. method:: str.encode(encoding="utf-8", errors="strict") From 81675941fb51f09b89ba46bce237222a87ea60f6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 12:00:38 +0200 Subject: [PATCH 217/713] [3.14] gh-133885: Use locks instead of critical sections for _zstd (gh-134289) (gh-134560) Move from using critical sections to locks for the (de)compression methods. Since the methods allow other threads to run, we should use a lock rather than a critical section. (cherry picked from commit 8dbc11971974a725dc8a11c0dc65d8f6fcb4d902) Co-authored-by: Emma Smith --- Lib/test/test_zstd.py | 56 +++++++++- Modules/_zstd/clinic/decompressor.c.h | 11 +- Modules/_zstd/clinic/zstddict.c.h | 27 +---- Modules/_zstd/compressor.c | 151 ++++++++++++++------------ Modules/_zstd/decompressor.c | 150 +++++++++++++------------ Modules/_zstd/zstddict.c | 13 ++- Modules/_zstd/zstddict.h | 3 + 7 files changed, 229 insertions(+), 182 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 53ca592ea38828..084f8f24fc009c 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -2430,10 +2430,8 @@ def test_buffer_protocol(self): self.assertEqual(f.write(arr), LENGTH) self.assertEqual(f.tell(), LENGTH) -@unittest.skip("it fails for now, see gh-133885") class FreeThreadingMethodTests(unittest.TestCase): - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_compress_locking(self): @@ -2470,7 +2468,6 @@ def run_method(method, input_data, output_data): actual = b''.join(output) + rest2 self.assertEqual(expected, actual) - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_decompress_locking(self): @@ -2506,6 +2503,59 @@ def run_method(method, input_data, output_data): actual = b''.join(output) self.assertEqual(expected, actual) + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_shared_dict(self): + num_threads = 8 + + def run_method(b): + level = threading.get_ident() % 4 + # sync threads to increase chance of contention on + # capsule storing dictionary levels + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_shared_dict(self): + num_threads = 8 + + def run_method(b): + # sync threads to increase chance of contention on + # decompression dictionary + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass if __name__ == "__main__": diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 4ecb19e9bde6ed..c6fdae74ab0447 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -7,7 +7,6 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, @@ -114,13 +113,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self); static PyObject * _zstd_ZstdDecompressor_unused_data_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); } PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, @@ -227,4 +220,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=30c12ef047027ede input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 34e0e4b3ecfe72..aaa29e491bc1bb 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -6,7 +6,6 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, @@ -118,13 +117,7 @@ _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, @@ -156,13 +149,7 @@ _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, @@ -194,12 +181,6 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8692eabee4e0d1fe input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 38baee2be1e95b..8f934858ef784f 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -17,6 +17,7 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" #include "_zstdmodule.h" #include "buffer.h" #include "zstddict.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked #include // offsetof() #include // ZSTD_*() @@ -38,6 +39,9 @@ typedef struct { /* Compression level */ int compression_level; + + /* Lock to protect the compression context */ + PyMutex lock; } ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) @@ -149,12 +153,12 @@ capsule_free_cdict(PyObject *capsule) ZSTD_CDict * _get_CDict(ZstdDict *self, int compressionLevel) { + assert(PyMutex_IsLocked(&self->lock)); PyObject *level = NULL; - PyObject *capsule; + PyObject *capsule = NULL; ZSTD_CDict *cdict; + int ret; - // TODO(emmatyping): refactor critical section code into a lock_held function - Py_BEGIN_CRITICAL_SECTION(self); /* int level object */ level = PyLong_FromLong(compressionLevel); @@ -163,12 +167,11 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Get PyCapsule object from self->c_dicts */ - capsule = PyDict_GetItemWithError(self->c_dicts, level); + ret = PyDict_GetItemRef(self->c_dicts, level, &capsule); + if (ret < 0) { + goto error; + } if (capsule == NULL) { - if (PyErr_Occurred()) { - goto error; - } - /* Create ZSTD_CDict instance */ char *dict_buffer = PyBytes_AS_STRING(self->dict_content); Py_ssize_t dict_len = Py_SIZE(self->dict_content); @@ -196,11 +199,10 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Add PyCapsule object to self->c_dicts */ - if (PyDict_SetItem(self->c_dicts, level, capsule) < 0) { - Py_DECREF(capsule); + ret = PyDict_SetItem(self->c_dicts, level, capsule); + if (ret < 0) { goto error; } - Py_DECREF(capsule); } else { /* ZSTD_CDict instance already exists */ @@ -212,15 +214,55 @@ _get_CDict(ZstdDict *self, int compressionLevel) cdict = NULL; success: Py_XDECREF(level); - Py_END_CRITICAL_SECTION(); + Py_XDECREF(capsule); return cdict; } static int -_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) +_zstd_load_impl(ZstdCompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) { - size_t zstd_ret; + if (type == DICT_TYPE_DIGESTED) { + /* Get ZSTD_CDict */ + ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level); + if (c_dict == NULL) { + return -1; + } + /* Reference a prepared dictionary. + It overrides some compression context's parameters. */ + zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict); + } + else if (type == DICT_TYPE_UNDIGESTED) { + /* Load a dictionary. + It doesn't override compression context's parameters. */ + zstd_ret = ZSTD_CCtx_loadDictionary( + self->cctx, + PyBytes_AS_STRING(zd->dict_content), + Py_SIZE(zd->dict_content)); + } + else if (type == DICT_TYPE_PREFIX) { + /* Load a prefix */ + zstd_ret = ZSTD_CCtx_refPrefix( + self->cctx, + PyBytes_AS_STRING(zd->dict_content), + Py_SIZE(zd->dict_content)); + } + else { + Py_UNREACHABLE(); + } + + /* Check error */ + if (ZSTD_isError(zstd_ret)) { + set_zstd_error(mod_state, ERR_LOAD_C_DICT, zstd_ret); + return -1; + } + return 0; +} + +static int +_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) +{ _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; @@ -237,7 +279,10 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) /* When compressing, use undigested dictionary by default. */ zd = (ZstdDict*)dict; type = DICT_TYPE_UNDIGESTED; - goto load; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } /* Check (ZstdDict, type) */ @@ -257,7 +302,10 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { assert(type >= 0); zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } } } @@ -266,49 +314,6 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) PyErr_SetString(PyExc_TypeError, "zstd_dict argument should be ZstdDict object."); return -1; - -load: - if (type == DICT_TYPE_DIGESTED) { - /* Get ZSTD_CDict */ - ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level); - if (c_dict == NULL) { - return -1; - } - /* Reference a prepared dictionary. - It overrides some compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION(self); - zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict); - Py_END_CRITICAL_SECTION(); - } - else if (type == DICT_TYPE_UNDIGESTED) { - /* Load a dictionary. - It doesn't override compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_loadDictionary( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); - } - else if (type == DICT_TYPE_PREFIX) { - /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_refPrefix( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); - } - else { - Py_UNREACHABLE(); - } - - /* Check error */ - if (ZSTD_isError(zstd_ret)) { - set_zstd_error(mod_state, ERR_LOAD_C_DICT, zstd_ret); - return -1; - } - return 0; } /*[clinic input] @@ -339,6 +344,7 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, self->use_multithread = 0; self->dict = NULL; + self->lock = (PyMutex){0}; /* Compression context */ self->cctx = ZSTD_createCCtx(); @@ -403,6 +409,8 @@ ZstdCompressor_dealloc(PyObject *ob) ZSTD_freeCCtx(self->cctx); } + assert(!PyMutex_IsLocked(&self->lock)); + /* Py_XDECREF the dict after free the compression context */ Py_CLEAR(self->dict); @@ -412,9 +420,10 @@ ZstdCompressor_dealloc(PyObject *ob) } static PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive) +compress_lock_held(ZstdCompressor *self, Py_buffer *data, + ZSTD_EndDirective end_directive) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; _BlocksOutputBuffer buffer = {.list = NULL}; @@ -495,8 +504,9 @@ mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) #endif static PyObject * -compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) +compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; _BlocksOutputBuffer buffer = {.list = NULL}; @@ -529,7 +539,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) goto error; } - /* Like compress_impl(), output as much as possible. */ + /* Like compress_lock_held(), output as much as possible. */ if (out.pos == out.size) { if (_OutputBuffer_Grow(&buffer, &out) < 0) { goto error; @@ -588,14 +598,14 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); + PyMutex_Lock(&self->lock); /* Compress */ if (self->use_multithread && mode == ZSTD_e_continue) { - ret = compress_mt_continue_impl(self, data); + ret = compress_mt_continue_lock_held(self, data); } else { - ret = compress_impl(self, data, mode); + ret = compress_lock_held(self, data, mode); } if (ret) { @@ -607,7 +617,7 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } @@ -642,8 +652,9 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - ret = compress_impl(self, NULL, mode); + PyMutex_Lock(&self->lock); + + ret = compress_lock_held(self, NULL, mode); if (ret) { self->last_mode = mode; @@ -654,7 +665,7 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 58f9c9f804e549..e299f73b071353 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -17,6 +17,7 @@ class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" #include "_zstdmodule.h" #include "buffer.h" #include "zstddict.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked #include // bool #include // offsetof() @@ -45,6 +46,9 @@ typedef struct { /* For ZstdDecompressor, 0 or 1. 1 means the end of the first frame has been reached. */ bool eof; + + /* Lock to protect the decompression context */ + PyMutex lock; } ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) @@ -54,6 +58,7 @@ typedef struct { static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_DDict *ret; /* Already created */ @@ -61,15 +66,14 @@ _get_DDict(ZstdDict *self) return self->d_dict; } - Py_BEGIN_CRITICAL_SECTION(self); if (self->d_dict == NULL) { /* Create ZSTD_DDict instance from dictionary content */ char *dict_buffer = PyBytes_AS_STRING(self->dict_content); Py_ssize_t dict_len = Py_SIZE(self->dict_content); Py_BEGIN_ALLOW_THREADS - self->d_dict = ZSTD_createDDict(dict_buffer, - dict_len); + ret = ZSTD_createDDict(dict_buffer, dict_len); Py_END_ALLOW_THREADS + self->d_dict = ret; if (self->d_dict == NULL) { _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); @@ -81,11 +85,7 @@ _get_DDict(ZstdDict *self) } } - /* Don't lose any exception */ - ret = self->d_dict; - Py_END_CRITICAL_SECTION(); - - return ret; + return self->d_dict; } /* Set decompression parameters to decompression context */ @@ -134,9 +134,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) } /* Set parameter to compression context */ - Py_BEGIN_CRITICAL_SECTION(self); zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); - Py_END_CRITICAL_SECTION(); /* Check error */ if (ZSTD_isError(zstd_ret)) { @@ -147,11 +145,53 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) return 0; } +static int +_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) +{ + size_t zstd_ret; + if (type == DICT_TYPE_DIGESTED) { + /* Get ZSTD_DDict */ + ZSTD_DDict *d_dict = _get_DDict(zd); + if (d_dict == NULL) { + return -1; + } + /* Reference a prepared dictionary */ + zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict); + } + else if (type == DICT_TYPE_UNDIGESTED) { + /* Load a dictionary */ + zstd_ret = ZSTD_DCtx_loadDictionary( + self->dctx, + PyBytes_AS_STRING(zd->dict_content), + Py_SIZE(zd->dict_content)); + } + else if (type == DICT_TYPE_PREFIX) { + /* Load a prefix */ + zstd_ret = ZSTD_DCtx_refPrefix( + self->dctx, + PyBytes_AS_STRING(zd->dict_content), + Py_SIZE(zd->dict_content)); + } + else { + /* Impossible code path */ + PyErr_SetString(PyExc_SystemError, + "load_d_dict() impossible code path"); + return -1; + } + + /* Check error */ + if (ZSTD_isError(zstd_ret)) { + set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret); + return -1; + } + return 0; +} + /* Load dictionary or prefix to decompression context */ static int _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { - size_t zstd_ret; _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; @@ -168,7 +208,10 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) /* When decompressing, use digested dictionary by default. */ zd = (ZstdDict*)dict; type = DICT_TYPE_DIGESTED; - goto load; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } /* Check (ZstdDict, type) */ @@ -188,7 +231,10 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { assert(type >= 0); zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; } } } @@ -197,50 +243,6 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) PyErr_SetString(PyExc_TypeError, "zstd_dict argument should be ZstdDict object."); return -1; - -load: - if (type == DICT_TYPE_DIGESTED) { - /* Get ZSTD_DDict */ - ZSTD_DDict *d_dict = _get_DDict(zd); - if (d_dict == NULL) { - return -1; - } - /* Reference a prepared dictionary */ - Py_BEGIN_CRITICAL_SECTION(self); - zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict); - Py_END_CRITICAL_SECTION(); - } - else if (type == DICT_TYPE_UNDIGESTED) { - /* Load a dictionary */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_loadDictionary( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); - } - else if (type == DICT_TYPE_PREFIX) { - /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_refPrefix( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); - } - else { - /* Impossible code path */ - PyErr_SetString(PyExc_SystemError, - "load_d_dict() impossible code path"); - return -1; - } - - /* Check error */ - if (ZSTD_isError(zstd_ret)) { - set_zstd_error(mod_state, ERR_LOAD_D_DICT, zstd_ret); - return -1; - } - return 0; } /* @@ -268,8 +270,8 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) Note, decompressing "an empty input" in any case will make it > 0. */ static PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length) +decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, + Py_ssize_t max_length) { size_t zstd_ret; ZSTD_outBuffer out; @@ -339,10 +341,9 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, } static void -decompressor_reset_session(ZstdDecompressor *self) +decompressor_reset_session_lock_held(ZstdDecompressor *self) { - // TODO(emmatyping): use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here - // and ensure lock is always held + assert(PyMutex_IsLocked(&self->lock)); /* Reset variables */ self->in_begin = 0; @@ -359,8 +360,10 @@ decompressor_reset_session(ZstdDecompressor *self) } static PyObject * -stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) +stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, + Py_ssize_t max_length) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; PyObject *ret = NULL; int use_input_buffer; @@ -456,7 +459,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length assert(in.pos == 0); /* Decompress */ - ret = decompress_impl(self, &in, max_length); + ret = decompress_lock_held(self, &in, max_length); if (ret == NULL) { goto error; } @@ -517,7 +520,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length error: /* Reset decompressor's states/session */ - decompressor_reset_session(self); + decompressor_reset_session_lock_held(self); Py_CLEAR(ret); return NULL; @@ -555,6 +558,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, self->unused_data = NULL; self->eof = 0; self->dict = NULL; + self->lock = (PyMutex){0}; /* needs_input flag */ self->needs_input = 1; @@ -608,6 +612,8 @@ ZstdDecompressor_dealloc(PyObject *ob) ZSTD_freeDCtx(self->dctx); } + assert(!PyMutex_IsLocked(&self->lock)); + /* Py_CLEAR the dict after free decompression context */ Py_CLEAR(self->dict); @@ -623,7 +629,6 @@ ZstdDecompressor_dealloc(PyObject *ob) } /*[clinic input] -@critical_section @getter _zstd.ZstdDecompressor.unused_data @@ -635,11 +640,14 @@ decompressed, unused input data after the frame. Otherwise this will be b''. static PyObject * _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) -/*[clinic end generated code: output=f3a20940f11b6b09 input=5233800bef00df04]*/ +/*[clinic end generated code: output=f3a20940f11b6b09 input=54d41ecd681a3444]*/ { PyObject *ret; + PyMutex_Lock(&self->lock); + if (!self->eof) { + PyMutex_Unlock(&self->lock); return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else { @@ -656,6 +664,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) } } + PyMutex_Unlock(&self->lock); return ret; } @@ -693,10 +702,9 @@ _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, { PyObject *ret; /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - - ret = stream_decompress(self, data, max_length); - Py_END_CRITICAL_SECTION(); + PyMutex_Lock(&self->lock); + ret = stream_decompress_lock_held(self, data, max_length); + PyMutex_Unlock(&self->lock); return ret; } diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 7df187a6fa69d7..39828c9b36b5c2 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -17,6 +17,7 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #include "_zstdmodule.h" #include "zstddict.h" #include "clinic/zstddict.c.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked #include // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() @@ -53,6 +54,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, self->dict_content = NULL; self->d_dict = NULL; self->dict_id = 0; + self->lock = (PyMutex){0}; /* ZSTD_CDict dict */ self->c_dicts = PyDict_New(); @@ -109,6 +111,8 @@ ZstdDict_dealloc(PyObject *ob) ZSTD_freeDDict(self->d_dict); } + assert(!PyMutex_IsLocked(&self->lock)); + /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ Py_CLEAR(self->dict_content); Py_CLEAR(self->c_dicts); @@ -143,7 +147,6 @@ static PyMemberDef ZstdDict_members[] = { }; /*[clinic input] -@critical_section @getter _zstd.ZstdDict.as_digested_dict @@ -160,13 +163,12 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digeste static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=585448c79f31f74a]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=10cd2b6165931b77]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } /*[clinic input] -@critical_section @getter _zstd.ZstdDict.as_undigested_dict @@ -181,13 +183,12 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undiges static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=022b0829ffb1c220]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=11e5f5df690a85b4]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } /*[clinic input] -@critical_section @getter _zstd.ZstdDict.as_prefix @@ -202,7 +203,7 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=09fb82a6a5407e87]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=b028e0ae6ec4292b]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index e8a55a3670b869..dcba0f21852087 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -19,6 +19,9 @@ typedef struct { PyObject *dict_content; /* Dictionary id */ uint32_t dict_id; + + /* Lock to protect the digested dictionaries */ + PyMutex lock; } ZstdDict; #endif // !ZSTD_DICT_H From adb0794692f3c4b711da09bd541fb8ce4faeb101 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 12:23:06 +0200 Subject: [PATCH 218/713] [3.14] Further improves Advanced installation docs for PyManager (GH-134541) (GH-134576) Further improves Advanced installation docs for PyManager (GH-134541) (cherry picked from commit 99a9ab1c64dce26c0f2dce7621b4d7f75da69856) Co-authored-by: Steve Dower --- Doc/using/windows.rst | 47 +++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index cd22148b531b84..9628da3d2f6b12 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -504,6 +504,14 @@ configuration option. installing and uninstalling. +.. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage + +.. _Remove-AppxPackage: https://learn.microsoft.com/powershell/module/appx/remove-appxpackage + +.. _Add-AppxProvisionedPackage: https://learn.microsoft.com/powershell/module/dism/add-appxprovisionedpackage + +.. _PackageManager: https://learn.microsoft.com/uwp/api/windows.management.deployment.packagemanager + .. _pymanager-advancedinstall: Advanced Installation @@ -559,12 +567,10 @@ downloaded MSIX can be installed by launching or using the commands below. .. code-block:: powershell - $> winget download 9NQ7512CXL7T -e --skip-license --accept-package-agreements --disable-interactivity + $> winget download 9NQ7512CXL7T -e --skip-license --accept-package-agreements --accept-source-agreements To programmatically install or uninstall an MSIX using only PowerShell, the -`Add-AppxPackage `_ -and `Remove-AppxPackage `_ -PowerShell cmdlets are recommended: +`Add-AppxPackage`_ and `Remove-AppxPackage`_ PowerShell cmdlets are recommended: .. code-block:: powershell @@ -572,18 +578,33 @@ PowerShell cmdlets are recommended: ... $> Get-AppxPackage PythonSoftwareFoundation.PythonManager | Remove-AppxPackage -The native APIs for package management may be found on the Windows -`PackageManager `_ -class. The :func:`!AddPackageAsync` method installs for the current user, or use -:func:`!StagePackageAsync` followed by :func:`!ProvisionPackageForAllUsersAsync` -to install the Python install manager for all users from the MSIX package. Users -will still need to install their own copies of Python itself, as there is no way -to trigger those installs without being a logged in user. +The latest release can be downloaded and installed by Windows by passing the +AppInstaller file to the Add-AppxPackage command. This installs using the MSIX +on python.org, and is only recommended for cases where installing via the Store +(interactively or using WinGet) is not possible. + +.. code-block:: powershell + + $> Add-AppxPackage -AppInstallerFile https://www.python.org/ftp/python/pymanager/pymanager.appinstaller + +Other tools and APIs may also be used to provision an MSIX package for all users +on a machine, but Python does not consider this a supported scenario. We suggest +looking into the PowerShell `Add-AppxProvisionedPackage`_ cmdlet, the native +Windows `PackageManager`_ class, or the documentation and support for your +deployment tool. + +Regardless of the install method, users will still need to install their own +copies of Python itself, as there is no way to trigger those installs without +being a logged in user. When using the MSIX, the latest version of Python will +be available for all users to install without network access. Note that the MSIX downloadable from the Store and from the Python website are subtly different and cannot be installed at the same time. Wherever possible, -we suggest using the above commands to download the package from the Store to -reduce the risk of setting up conflicting installs. +we suggest using the above WinGet commands to download the package from the +Store to reduce the risk of setting up conflicting installs. There are no +licensing restrictions on the Python install manager that would prevent using +the Store package in this way. + .. _pymanager-admin-config: From 6e605861751eb0f8cca3836e2eadd34faf6a7ad2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 19:23:47 +0200 Subject: [PATCH 219/713] [3.14] gh-133454: Reduce the number of threads in test_racing_getbuf_and_releasebuf (GH-133458) (GH-134589) The original reproducer only used 10 threads. (cherry picked from commit fc0c9c24121c1a62b25b79062286f976699b59e9) Co-authored-by: Serhiy Storchaka --- Lib/test/test_memoryview.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 61b068c630c7ce..64f440f180bbf0 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -743,19 +743,21 @@ def test_racing_getbuf_and_releasebuf(self): from multiprocessing.managers import SharedMemoryManager except ImportError: self.skipTest("Test requires multiprocessing") - from threading import Thread + from threading import Thread, Event - n = 100 + start = Event() with SharedMemoryManager() as smm: obj = smm.ShareableList(range(100)) - threads = [] - for _ in range(n): - # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` - # counter of `memoryview`, this issue has no direct relation with `ShareableList`. - threads.append(Thread(target=obj.count, args=(1,))) - + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] with threading_helper.start_threads(threads): - pass + start.set() del obj From 09a34f1f659cd66c380353cf9b8b0247789128cc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 23 May 2025 21:49:13 +0200 Subject: [PATCH 220/713] [3.14] gh-134381: Fix RuntimeError when starting not-yet started Thread after fork (gh-134514) (gh-134596) (cherry picked from commit 9a2346df861f26d5f8d054ad2f9c37134dee3822) Co-authored-by: Jiucheng(Oliver) --- Lib/test/_test_multiprocessing.py | 22 +++++++++++++++++++ ...-05-22-14-48-19.gh-issue-134381.2BXhth.rst | 1 + Modules/_threadmodule.c | 6 +++++ 3 files changed, 29 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 8cc4e46fa12a68..63f522f6ecce21 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -6832,6 +6832,28 @@ def f(x): return x*x self.assertEqual("332833500", out.decode('utf-8').strip()) self.assertFalse(err, msg=err.decode('utf-8')) + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + # # Mixins diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst new file mode 100644 index 00000000000000..aa8900296ae2fc --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst @@ -0,0 +1 @@ +Fix :exc:`RuntimeError` when using a not-started :class:`threading.Thread` after calling :func:`os.fork` diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index f676659af369f6..73739d24f33c05 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -281,6 +281,12 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) continue; } + // Keep handles for threads that have not been started yet. They are + // safe to start in the child process. + if (handle->state == THREAD_HANDLE_NOT_STARTED) { + continue; + } + // Mark all threads as done. Any attempts to join or detach the // underlying OS thread (if any) could crash. We are the only thread; // it's safe to set this non-atomically. From 7476f90af2cf9207c792c0dc3ee9a5ca7b4fabfa Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 23 May 2025 14:11:38 -0600 Subject: [PATCH 221/713] [3.14] gh-134557: Revert "[3.14] gh-132775: Use _PyCode GetScriptXIData()" (gh-134600) This reverts commit bbf8048c0f9c, AKA gh-134515. We are reverting due to refleaks on free-threaded builds. --- Lib/test/support/interpreters/channels.py | 2 +- Lib/test/test__interpreters.py | 25 +- Lib/test/test_interpreters/test_api.py | 23 +- Modules/_interpretersmodule.c | 294 ++++++++++++++++------ 4 files changed, 240 insertions(+), 104 deletions(-) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py index 3b6e0f0effd969..7a2bd7d63f808f 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/interpreters/channels.py @@ -69,7 +69,7 @@ def list_all(): if not hasattr(send, '_unboundop'): send._set_unbound(unboundop) else: - assert send._unbound[0] == unboundop + assert send._unbound[0] == op channels.append(chan) return channels diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index ad3ebbfdff64a7..63fdaad8de7ef5 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,15 +474,13 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = r'_interpreters.exec\(\) argument 3 must be dict, not int' + msg = "expected 'shared' to be a dict" with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) - msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) - msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) @@ -954,8 +952,7 @@ def test_invalid_syntax(self): """) with self.subTest('script'): - with self.assertRaises(SyntaxError): - _interpreters.run_string(self.id, script) + self.assert_run_failed(SyntaxError, script) with self.subTest('module'): modname = 'spam_spam_spam' @@ -1022,19 +1019,12 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') - failed = None def f(): - nonlocal failed - try: - _interpreters.set___main___attrs(self.id, dict(w=w)) - _interpreters.run_func(self.id, script) - except Exception as exc: - failed = exc + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) t = threading.Thread(target=f) t.start() t.join() - if failed: - raise Exception from failed with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1063,16 +1053,19 @@ def test_closure(self): spam = True def script(): assert spam - with self.assertRaises(ValueError): + + with self.assertRaises(TypeError): _interpreters.run_func(self.id, script) + # XXX This hasn't been fixed yet. + @unittest.expectedFailure def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) -# @unittest.skip("we're not quite there yet") + @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 165949167ceba8..1e2d572b1cbb81 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -839,16 +839,9 @@ def test_bad_script(self): interp.exec(10) def test_bytes_for_script(self): - r, w = self.pipe() - RAN = b'R' - DONE = b'D' interp = interpreters.create() - interp.exec(f"""if True: - import os - os.write({w}, {RAN!r}) - """) - os.write(w, DONE) - self.assertEqual(os.read(r, 1), RAN) + with self.assertRaises(TypeError): + interp.exec(b'print("spam")') def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -1017,6 +1010,8 @@ def test_call(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1041,8 +1036,6 @@ def test_call(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): with self.assertRaises(Exception): @@ -1058,6 +1051,8 @@ def test_call_in_thread(self): for i, (callable, args, kwargs) in enumerate([ (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_not_shareable, (), {}), (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): @@ -1084,8 +1079,6 @@ def test_call_in_thread(self): (call_func_complex, ('custom', 'spam!'), {}), (call_func_complex, ('custom-inner', 'eggs!'), {}), (call_func_complex, ('???',), {'exc': ValueError('spam')}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), ]): with self.subTest(f'invalid case #{i+1}'): if args or kwargs: @@ -1625,8 +1618,8 @@ def test_exec(self): def test_call(self): with self.subTest('no args'): interpid = _interpreters.create() - with self.assertRaises(ValueError): - _interpreters.call(interpid, call_func_return_shareable) + exc = _interpreters.call(interpid, call_func_return_shareable) + self.assertIs(exc, None) with self.subTest('uncaught exception'): interpid = _interpreters.create() diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 376517ab92360f..f4807fd214b19e 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -9,6 +9,7 @@ #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() +#include "pycore_function.h" // _PyFunction_VerifyStateless() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -360,6 +361,81 @@ _get_current_xibufferview_type(void) } +/* Python code **************************************************************/ + +static const char * +check_code_str(PyUnicodeObject *text) +{ + assert(text != NULL); + if (PyUnicode_GET_LENGTH(text) == 0) { + return "too short"; + } + + // XXX Verify that it parses? + + return NULL; +} + +#ifndef NDEBUG +static int +code_has_args(PyCodeObject *code) +{ + assert(code != NULL); + return (code->co_argcount > 0 + || code->co_posonlyargcount > 0 + || code->co_kwonlyargcount > 0 + || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)); +} +#endif + +#define RUN_TEXT 1 +#define RUN_CODE 2 + +static const char * +get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) +{ + const char *codestr = NULL; + Py_ssize_t len = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + + if (PyUnicode_Check(arg)) { + assert(PyUnicode_Check(arg) + && (check_code_str((PyUnicodeObject *)arg) == NULL)); + codestr = PyUnicode_AsUTF8AndSize(arg, &len); + if (codestr == NULL) { + return NULL; + } + if (strlen(codestr) != (size_t)len) { + PyErr_SetString(PyExc_ValueError, + "source code string cannot contain null bytes"); + return NULL; + } + flags = RUN_TEXT; + } + else { + assert(PyCode_Check(arg)); + assert(_PyCode_VerifyStateless( + PyThreadState_Get(), (PyCodeObject *)arg, NULL, NULL, NULL) == 0); + assert(!code_has_args((PyCodeObject *)arg)); + flags = RUN_CODE; + + // Serialize the code object. + bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); + if (bytes_obj == NULL) { + return NULL; + } + codestr = PyBytes_AS_STRING(bytes_obj); + len = PyBytes_GET_SIZE(bytes_obj); + } + + *flags_p = flags; + *bytes_p = bytes_obj; + *len_p = len; + return codestr; +} + + /* interpreter-specific code ************************************************/ static int @@ -423,14 +499,22 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) static int -_run_script(_PyXIData_t *script, PyObject *ns) +_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) { - PyObject *code = _PyXIData_NewObject(script); - if (code == NULL) { - return -1; + PyObject *result = NULL; + if (flags & RUN_TEXT) { + result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); + } + else if (flags & RUN_CODE) { + PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); + if (code != NULL) { + result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + } + } + else { + Py_UNREACHABLE(); } - PyObject *result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); if (result == NULL) { return -1; } @@ -439,11 +523,12 @@ _run_script(_PyXIData_t *script, PyObject *ns) } static int -_exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, - _PyXIData_t *script, PyObject *shareables, +_run_in_interpreter(PyInterpreterState *interp, + const char *codestr, Py_ssize_t codestrlen, + PyObject *shareables, int flags, PyObject **p_excinfo) { - assert(!_PyErr_Occurred(tstate)); + assert(!PyErr_Occurred()); _PyXI_session *session = _PyXI_NewSession(); if (session == NULL) { return -1; @@ -451,7 +536,7 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables) < 0) { - if (_PyErr_Occurred(tstate)) { + if (PyErr_Occurred()) { // If an error occured at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); @@ -473,7 +558,7 @@ _exec_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, if (mainns == NULL) { goto finally; } - res = _run_script(script, mainns); + res = _run_script(mainns, codestr, codestrlen, flags); finally: // Clean up and switch back. @@ -867,23 +952,104 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static void -unwrap_not_shareable(PyThreadState *tstate) +static PyUnicodeObject * +convert_script_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, + const char *expected) { - PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); - if (!_PyErr_ExceptionMatches(tstate, exctype)) { - return; + PyUnicodeObject *str = NULL; + if (PyUnicode_CheckExact(arg)) { + str = (PyUnicodeObject *)Py_NewRef(arg); } - PyObject *exc = _PyErr_GetRaisedException(tstate); - PyObject *cause = PyException_GetCause(exc); - if (cause != NULL) { - Py_DECREF(exc); - exc = cause; + else if (PyUnicode_Check(arg)) { + // XXX str = PyUnicode_FromObject(arg); + str = (PyUnicodeObject *)Py_NewRef(arg); } else { - assert(PyException_GetContext(exc) == NULL); + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; } + + const char *err = check_code_str(str); + if (err != NULL) { + Py_DECREF(str); + _PyErr_Format(tstate, PyExc_ValueError, + "%.200s(): bad script text (%s)", fname, err); + return NULL; + } + + return str; +} + +static PyCodeObject * +convert_code_arg(PyThreadState *tstate, + PyObject *arg, const char *fname, const char *displayname, + const char *expected) +{ + PyObject *cause; + PyCodeObject *code = NULL; + if (PyFunction_Check(arg)) { + // For now we allow globals, so we can't use + // _PyFunction_VerifyStateless(). + PyObject *codeobj = PyFunction_GetCode(arg); + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)codeobj, NULL, NULL, NULL) < 0) { + goto chained; + } + code = (PyCodeObject *)Py_NewRef(codeobj); + } + else if (PyCode_Check(arg)) { + if (_PyCode_VerifyStateless( + tstate, (PyCodeObject *)arg, NULL, NULL, NULL) < 0) { + goto chained; + } + code = (PyCodeObject *)Py_NewRef(arg); + } + else { + _PyArg_BadArgument(fname, displayname, expected, arg); + return NULL; + } + + return code; + +chained: + cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + _PyArg_BadArgument(fname, displayname, expected, arg); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); _PyErr_SetRaisedException(tstate, exc); + return NULL; +} + +static int +_interp_exec(PyObject *self, PyInterpreterState *interp, + PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) +{ + if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { + PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); + return -1; + } + + // Extract code. + Py_ssize_t codestrlen = -1; + PyObject *bytes_obj = NULL; + int flags = 0; + const char *codestr = get_code_str(code_arg, + &codestrlen, &bytes_obj, &flags); + if (codestr == NULL) { + return -1; + } + + // Run the code in the interpreter. + int res = _run_in_interpreter(interp, codestr, codestrlen, + shared_arg, flags, p_excinfo); + Py_XDECREF(bytes_obj); + if (res < 0) { + return -1; + } + + return 0; } static PyObject * @@ -896,9 +1062,8 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &code, &PyDict_Type, &shared, - &restricted)) + "OO|O$p:" FUNCNAME, kwlist, + &id, &code, &shared, &restricted)) { return NULL; } @@ -910,17 +1075,22 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // We don't need the script to be "pure", which means it can use - // global variables. They will be resolved against __main__. - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + const char *expected = "a string, a function, or a code object"; + if (PyUnicode_Check(code)) { + code = (PyObject *)convert_script_arg(tstate, code, FUNCNAME, + "argument 2", expected); + } + else { + code = (PyObject *)convert_code_arg(tstate, code, FUNCNAME, + "argument 2", expected); + } + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, code, shared, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -956,9 +1126,8 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O!$p:" FUNCNAME, kwlist, - &id, &script, &PyDict_Type, &shared, - &restricted)) + "OU|O$p:" FUNCNAME, kwlist, + &id, &script, &shared, &restricted)) { return NULL; } @@ -970,20 +1139,15 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (PyFunction_Check(script) || PyCode_Check(script)) { - _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); - return NULL; - } - - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { - unwrap_not_shareable(tstate); + script = (PyObject *)convert_script_arg(tstate, script, FUNCNAME, + "argument 2", "a string"); + if (script == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, script, shared, &excinfo); + Py_DECREF(script); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1009,9 +1173,8 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O!$p:" FUNCNAME, kwlist, - &id, &func, &PyDict_Type, &shared, - &restricted)) + "OO|O$p:" FUNCNAME, kwlist, + &id, &func, &shared, &restricted)) { return NULL; } @@ -1023,29 +1186,16 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - // We don't worry about checking globals. They will be resolved - // against __main__. - PyObject *code; - if (PyFunction_Check(func)) { - code = PyFunction_GET_CODE(func); - } - else if (PyCode_Check(func)) { - code = func; - } - else { - _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); - return NULL; - } - - _PyXIData_t xidata = {0}; - if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { - unwrap_not_shareable(tstate); + PyCodeObject *code = convert_code_arg(tstate, func, FUNCNAME, + "argument 2", + "a function or a code object"); + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, shared, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; @@ -1098,15 +1248,15 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - _PyXIData_t xidata = {0}; - if (_PyCode_GetPureScriptXIData(tstate, callable, &xidata) < 0) { - unwrap_not_shareable(tstate); + PyObject *code = (PyObject *)convert_code_arg(tstate, callable, FUNCNAME, + "argument 2", "a function"); + if (code == NULL) { return NULL; } PyObject *excinfo = NULL; - int res = _exec_in_interpreter(tstate, interp, &xidata, NULL, &excinfo); - _PyXIData_Release(&xidata); + int res = _interp_exec(self, interp, code, NULL, &excinfo); + Py_DECREF(code); if (res < 0) { assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); return excinfo; From f301af627ea5138726a85b5d34ea92bf6198bcad Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 00:18:08 +0200 Subject: [PATCH 222/713] [3.14] gh-132983: Slightly tweak error messages for _zstd compressor/decompressor options dict (GH-134601) (#134602) gh-132983: Slightly tweak error messages for _zstd compressor/decompressor options dict (GH-134601) Slightly tweak error messages for options dict (cherry picked from commit f478331f98930d94f7efc741f3bed4b693d5cec1) Co-authored-by: Emma Smith --- Lib/test/test_zstd.py | 5 +++-- Modules/_zstd/compressor.c | 11 +++++------ Modules/_zstd/decompressor.c | 7 +++---- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 084f8f24fc009c..34c7c721b1ad32 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -1424,11 +1424,12 @@ def test_init_bad_mode(self): with self.assertRaises(ValueError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw") - with self.assertRaisesRegex(TypeError, r"NOT be CompressionParameter"): + with self.assertRaisesRegex(TypeError, + r"NOT be a CompressionParameter"): ZstdFile(io.BytesIO(), 'rb', options={CompressionParameter.compression_level:5}) with self.assertRaisesRegex(TypeError, - r"NOT be DecompressionParameter"): + r"NOT be a DecompressionParameter"): ZstdFile(io.BytesIO(), 'wb', options={DecompressionParameter.window_log_max:21}) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 8f934858ef784f..0f2967b60555a5 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -93,24 +93,23 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, /* Check key type */ if (Py_TYPE(key) == mod_state->DParameter_type) { PyErr_SetString(PyExc_TypeError, - "Key of compression option dict should " - "NOT be DecompressionParameter."); + "Key of compression options dict should " + "NOT be a DecompressionParameter attribute."); return -1; } int key_v = PyLong_AsInt(key); if (key_v == -1 && PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a CompressionParameter attribute."); + "Key of options dict should be either a " + "CompressionParameter attribute or an int."); return -1; } - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? int value_v = PyLong_AsInt(value); if (value_v == -1 && PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, - "Value of option dict should be an int."); + "Value of options dict should be an int."); return -1; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index e299f73b071353..70848d98534fa5 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -112,7 +112,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) if (Py_TYPE(key) == mod_state->CParameter_type) { PyErr_SetString(PyExc_TypeError, "Key of decompression options dict should " - "NOT be CompressionParameter."); + "NOT be a CompressionParameter attribute."); return -1; } @@ -120,12 +120,11 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) int key_v = PyLong_AsInt(key); if (key_v == -1 && PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a DecompressionParameter attribute."); + "Key of options dict should be either a " + "DecompressionParameter attribute or an int."); return -1; } - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? int value_v = PyLong_AsInt(value); if (value_v == -1 && PyErr_Occurred()) { PyErr_SetString(PyExc_ValueError, From 38a38243b7102e611c4dc3719975ff27dc7368b8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 04:28:40 +0200 Subject: [PATCH 223/713] [3.14] gh-132983: Make _zstd C code PEP 7 compliant (GH-134605) (#134609) gh-132983: Make _zstd C code PEP 7 compliant (GH-134605) Make _zstd C code PEP 7 compliant (cherry picked from commit 973b8f69d31b4d86c37e0b7194a209f4f2efff06) Co-authored-by: Emma Smith --- Modules/_zstd/_zstdmodule.c | 118 +++++++++++++++--------------- Modules/_zstd/buffer.h | 9 ++- Modules/_zstd/clinic/zstddict.c.h | 14 +++- Modules/_zstd/compressor.c | 43 ++++++----- Modules/_zstd/decompressor.c | 30 ++++---- Modules/_zstd/zstddict.c | 32 +++++--- 6 files changed, 137 insertions(+), 109 deletions(-) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 17d3bff1e98769..56ad999e5cd4e4 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -28,41 +28,42 @@ set_zstd_error(const _zstd_state* const state, char *msg; assert(ZSTD_isError(zstd_ret)); - switch (type) - { - case ERR_DECOMPRESS: - msg = "Unable to decompress Zstandard data: %s"; - break; - case ERR_COMPRESS: - msg = "Unable to compress Zstandard data: %s"; - break; - - case ERR_LOAD_D_DICT: - msg = "Unable to load Zstandard dictionary or prefix for decompression: %s"; - break; - case ERR_LOAD_C_DICT: - msg = "Unable to load Zstandard dictionary or prefix for compression: %s"; - break; - - case ERR_GET_C_BOUNDS: - msg = "Unable to get zstd compression parameter bounds: %s"; - break; - case ERR_GET_D_BOUNDS: - msg = "Unable to get zstd decompression parameter bounds: %s"; - break; - case ERR_SET_C_LEVEL: - msg = "Unable to set zstd compression level: %s"; - break; - - case ERR_TRAIN_DICT: - msg = "Unable to train the Zstandard dictionary: %s"; - break; - case ERR_FINALIZE_DICT: - msg = "Unable to finalize the Zstandard dictionary: %s"; - break; - - default: - Py_UNREACHABLE(); + switch (type) { + case ERR_DECOMPRESS: + msg = "Unable to decompress Zstandard data: %s"; + break; + case ERR_COMPRESS: + msg = "Unable to compress Zstandard data: %s"; + break; + + case ERR_LOAD_D_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "decompression: %s"; + break; + case ERR_LOAD_C_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "compression: %s"; + break; + + case ERR_GET_C_BOUNDS: + msg = "Unable to get zstd compression parameter bounds: %s"; + break; + case ERR_GET_D_BOUNDS: + msg = "Unable to get zstd decompression parameter bounds: %s"; + break; + case ERR_SET_C_LEVEL: + msg = "Unable to set zstd compression level: %s"; + break; + + case ERR_TRAIN_DICT: + msg = "Unable to train the Zstandard dictionary: %s"; + break; + case ERR_FINALIZE_DICT: + msg = "Unable to finalize the Zstandard dictionary: %s"; + break; + + default: + Py_UNREACHABLE(); } PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret)); } @@ -183,7 +184,7 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, chunks_number = Py_SIZE(samples_sizes); if ((size_t) chunks_number > UINT32_MAX) { PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); + "The number of samples should be <= %u.", UINT32_MAX); return -1; } @@ -200,8 +201,8 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, (*chunk_sizes)[i] = PyLong_AsSize_t(size); if ((*chunk_sizes)[i] == (size_t)-1 && PyErr_Occurred()) { PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); + "Items in samples_sizes should be an int " + "object, with a value between 0 and %u.", SIZE_MAX); return -1; } sizes_sum += (*chunk_sizes)[i]; @@ -209,7 +210,8 @@ calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, if (sizes_sum != Py_SIZE(samples_bytes)) { PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); + "The samples size tuple doesn't match the " + "concatenation's size."); return -1; } return chunks_number; @@ -242,15 +244,15 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); + PyErr_SetString(PyExc_ValueError, + "dict_size argument should be positive number."); return NULL; } /* Check that the samples are valid and get their sizes */ chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, &chunk_sizes); - if (chunks_number < 0) - { + if (chunks_number < 0) { goto error; } @@ -271,7 +273,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); goto error; } @@ -324,15 +326,15 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); + PyErr_SetString(PyExc_ValueError, + "dict_size argument should be positive number."); return NULL; } /* Check that the samples are valid and get their sizes */ chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, &chunk_sizes); - if (chunks_number < 0) - { + if (chunks_number < 0) { goto error; } @@ -355,14 +357,15 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_finalizeDictionary( PyBytes_AS_STRING(dst_dict_bytes), dict_size, - PyBytes_AS_STRING(custom_dict_bytes), Py_SIZE(custom_dict_bytes), + PyBytes_AS_STRING(custom_dict_bytes), + Py_SIZE(custom_dict_bytes), PyBytes_AS_STRING(samples_bytes), chunk_sizes, (uint32_t)chunks_number, params); Py_END_ALLOW_THREADS /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); goto error; } @@ -402,7 +405,7 @@ _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) if (is_compress) { bound = ZSTD_cParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_C_BOUNDS, bound.error); return NULL; } @@ -410,7 +413,7 @@ _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) else { bound = ZSTD_dParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_D_BOUNDS, bound.error); return NULL; } @@ -435,9 +438,10 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) { size_t frame_size; - frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, frame_buffer->len); + frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, + frame_buffer->len); if (ZSTD_isError(frame_size)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_Format(mod_state->ZstdError, "Error when finding the compressed size of a Zstandard frame. " "Ensure the frame_buffer argument starts from the " @@ -473,7 +477,7 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) /* #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) */ if (decompressed_size == ZSTD_CONTENTSIZE_ERROR) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_SetString(mod_state->ZstdError, "Error when getting information from the header of " "a Zstandard frame. Ensure the frame_buffer argument " @@ -508,7 +512,7 @@ _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, PyObject *d_parameter_type) /*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/ { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) { PyErr_SetString(PyExc_ValueError, @@ -568,7 +572,7 @@ do { \ Py_DECREF(v); \ } while (0) - _zstd_state* const mod_state = get_zstd_state(m); + _zstd_state* mod_state = get_zstd_state(m); /* Reusable objects & variables */ mod_state->CParameter_type = NULL; @@ -674,7 +678,7 @@ do { \ static int _zstd_traverse(PyObject *module, visitproc visit, void *arg) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); Py_VISIT(mod_state->ZstdDict_type); Py_VISIT(mod_state->ZstdCompressor_type); @@ -691,7 +695,7 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg) static int _zstd_clear(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); Py_CLEAR(mod_state->ZstdDict_type); Py_CLEAR(mod_state->ZstdCompressor_type); diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index bff3a81d8aa11c..4c885fa0d720fd 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -19,7 +19,8 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, /* Ensure .list was set to NULL */ assert(buffer->list == NULL); - Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, + &ob->dst); if (res < 0) { return -1; } @@ -34,8 +35,7 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, Return -1 on failure */ static inline int _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, - Py_ssize_t max_length, - Py_ssize_t init_size) + Py_ssize_t max_length, Py_ssize_t init_size) { Py_ssize_t block_size; @@ -50,7 +50,8 @@ _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, block_size = init_size; } - Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, + &ob->dst); if (res < 0) { return -1; } diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index aaa29e491bc1bb..810befdaf71f44 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -89,7 +89,9 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__, "Load as a digested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_digested_dict)\n" +"\n" "1. Some advanced compression parameters of compressor may be overridden\n" " by parameters of digested dictionary.\n" "2. ZstdDict has a digested dictionaries cache for each compression level.\n" @@ -123,7 +125,9 @@ _zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, "Load as an undigested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_undigested_dict)\n" +"\n" "1. The advanced compression parameters of compressor will not be overridden.\n" "2. Loading an undigested dictionary is costly. If load an undigested dictionary\n" " multiple times, consider reusing a compressor object.\n" @@ -155,7 +159,9 @@ _zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "Load as a prefix to compressor/decompressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_prefix)\n" +"\n" "1. Prefix is compatible with long distance matching, while dictionary is not.\n" "2. It only works for the first frame, then the compressor/decompressor will\n" " return to no prefix state.\n" @@ -183,4 +189,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=8692eabee4e0d1fe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=47b12b5848b53ed8 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 0f2967b60555a5..31cb8c535c05a6 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -53,7 +53,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, const char *arg_name, const char* arg_type) { size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } @@ -63,8 +63,8 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, int level = PyLong_AsInt(level_or_options); if (level == -1 && PyErr_Occurred()) { PyErr_Format(PyExc_ValueError, - "Compression level should be an int value between %d and %d.", - ZSTD_minCLevel(), ZSTD_maxCLevel()); + "Compression level should be an int value between " + "%d and %d.", ZSTD_minCLevel(), ZSTD_maxCLevel()); return -1; } @@ -138,7 +138,8 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, } return 0; } - PyErr_Format(PyExc_TypeError, "Invalid type for %s. Expected %s", arg_name, arg_type); + PyErr_Format(PyExc_TypeError, + "Invalid type for %s. Expected %s", arg_name, arg_type); return -1; } @@ -181,7 +182,7 @@ _get_CDict(ZstdDict *self, int compressionLevel) Py_END_ALLOW_THREADS if (cdict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Failed to create a ZSTD_CDict instance from " @@ -262,7 +263,7 @@ _zstd_load_impl(ZstdCompressor *self, ZstdDict *zd, static int _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } @@ -295,9 +296,9 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) else if (ret > 0) { /* type == -1 may indicate an error. */ type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) + if (type == DICT_TYPE_DIGESTED + || type == DICT_TYPE_UNDIGESTED + || type == DICT_TYPE_PREFIX) { assert(type >= 0); zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); @@ -348,7 +349,7 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, /* Compression context */ self->cctx = ZSTD_createCCtx(); if (self->cctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_CCtx instance."); @@ -360,7 +361,8 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, self->last_mode = ZSTD_e_end; if (level != Py_None && options != Py_None) { - PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); + PyErr_SetString(PyExc_RuntimeError, + "Only one of level or options should be used."); goto error; } @@ -449,7 +451,7 @@ compress_lock_held(ZstdCompressor *self, Py_buffer *data, } if (_OutputBuffer_InitWithSize(&buffer, &out, -1, - (Py_ssize_t) output_buffer_size) < 0) { + (Py_ssize_t) output_buffer_size) < 0) { goto error; } @@ -462,7 +464,7 @@ compress_lock_held(ZstdCompressor *self, Py_buffer *data, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); } @@ -525,13 +527,16 @@ compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) while (1) { Py_BEGIN_ALLOW_THREADS do { - zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, ZSTD_e_continue); - } while (out.pos != out.size && in.pos != in.size && !ZSTD_isError(zstd_ret)); + zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, + ZSTD_e_continue); + } while (out.pos != out.size + && in.pos != in.size + && !ZSTD_isError(zstd_ret)); Py_END_ALLOW_THREADS /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); } @@ -678,12 +683,12 @@ static PyMethodDef ZstdCompressor_methods[] = { PyDoc_STRVAR(ZstdCompressor_last_mode_doc, "The last mode used to this compressor object, its value can be .CONTINUE,\n" ".FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\n" -"It can be used to get the current state of a compressor, such as, data flushed,\n" -"a frame ended."); +"It can be used to get the current state of a compressor, such as, data\n" +"flushed, or a frame ended."); static PyMemberDef ZstdCompressor_members[] = { {"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode), - Py_READONLY, ZstdCompressor_last_mode_doc}, + Py_READONLY, ZstdCompressor_last_mode_doc}, {NULL} }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 70848d98534fa5..d084f0847c72dd 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -76,7 +76,7 @@ _get_DDict(ZstdDict *self) self->d_dict = ret; if (self->d_dict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Failed to create a ZSTD_DDict instance from " @@ -95,7 +95,7 @@ _zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) size_t zstd_ret; PyObject *key, *value; Py_ssize_t pos; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } @@ -191,7 +191,7 @@ _zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, static int _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } @@ -224,9 +224,9 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) else if (ret > 0) { /* type == -1 may indicate an error. */ type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) + if (type == DICT_TYPE_DIGESTED + || type == DICT_TYPE_UNDIGESTED + || type == DICT_TYPE_PREFIX) { assert(type >= 0); zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); @@ -291,7 +291,7 @@ decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); } @@ -369,7 +369,8 @@ stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, /* Check .eof flag */ if (self->eof) { - PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame."); + PyErr_SetString(PyExc_EOFError, + "Already at the end of a Zstandard frame."); assert(ret == NULL); return NULL; } @@ -486,8 +487,8 @@ stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, if (!use_input_buffer) { /* Discard buffer if it's too small (resizing it may needlessly copy the current contents) */ - if (self->input_buffer != NULL && - self->input_buffer_size < data_size) + if (self->input_buffer != NULL + && self->input_buffer_size < data_size) { PyMem_Free(self->input_buffer); self->input_buffer = NULL; @@ -565,7 +566,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, /* Decompression context */ self->dctx = ZSTD_createDCtx(); if (self->dctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_DCtx instance."); @@ -717,9 +718,10 @@ PyDoc_STRVAR(ZstdDecompressor_eof_doc, "after that, an EOFError exception will be raised."); PyDoc_STRVAR(ZstdDecompressor_needs_input_doc, -"If the max_length output limit in .decompress() method has been reached, and\n" -"the decompressor has (or may has) unconsumed input data, it will be set to\n" -"False. In this case, pass b'' to .decompress() method may output further data."); +"If the max_length output limit in .decompress() method has been reached,\n" +"and the decompressor has (or may has) unconsumed input data, it will be set\n" +"to False. In this case, passing b'' to the .decompress() method may output\n" +"further data."); static PyMemberDef ZstdDecompressor_members[] = { {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof), diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 39828c9b36b5c2..e3e9e5d064515f 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -74,13 +74,15 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, at least 8 bytes */ if (Py_SIZE(self->dict_content) < 8) { PyErr_SetString(PyExc_ValueError, - "Zstandard dictionary content should at least 8 bytes."); + "Zstandard dictionary content should at least " + "8 bytes."); goto error; } /* Get dict_id, 0 means "raw content" dictionary. */ - self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content), - Py_SIZE(self->dict_content)); + self->dict_id = ZSTD_getDictID_fromDict( + PyBytes_AS_STRING(self->dict_content), + Py_SIZE(self->dict_content)); /* Check validity for ordinary dictionary */ if (!is_raw && self->dict_id == 0) { @@ -141,8 +143,10 @@ ZstdDict_str(PyObject *ob) } static PyMemberDef ZstdDict_members[] = { - {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc}, - {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc}, + {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, + ZstdDict_dictid_doc}, + {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), + Py_READONLY, ZstdDict_dictcontent_doc}, {NULL} }; @@ -152,7 +156,9 @@ _zstd.ZstdDict.as_digested_dict Load as a digested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_digested_dict) + 1. Some advanced compression parameters of compressor may be overridden by parameters of digested dictionary. 2. ZstdDict has a digested dictionaries cache for each compression level. @@ -163,7 +169,7 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digeste static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=10cd2b6165931b77]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=ee45e1b4a48f6f2c]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } @@ -174,7 +180,9 @@ _zstd.ZstdDict.as_undigested_dict Load as an undigested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_undigested_dict) + 1. The advanced compression parameters of compressor will not be overridden. 2. Loading an undigested dictionary is costly. If load an undigested dictionary multiple times, consider reusing a compressor object. @@ -183,7 +191,7 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undiges static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=11e5f5df690a85b4]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=d39210eedec76fed]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } @@ -194,7 +202,9 @@ _zstd.ZstdDict.as_prefix Load as a prefix to compressor/decompressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_prefix) + 1. Prefix is compatible with long distance matching, while dictionary is not. 2. It only works for the first frame, then the compressor/decompressor will return to no prefix state. @@ -203,7 +213,7 @@ Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=b028e0ae6ec4292b]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=d59757b0b5a9551a]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } From 576177d4b3c9385a943c202d9ed0af7988f67341 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 05:33:35 +0200 Subject: [PATCH 224/713] [3.14] gh-128840: Limit the number of parts in IPv6 address parsing (GH-128841) (#134610) gh-128840: Limit the number of parts in IPv6 address parsing (GH-128841) GH-128840: Limit the number of parts in IPv6 address parsing Limit length of IP address string to 39 --------- (cherry picked from commit 47f1161d3a2bec52b5b5e952150141709c247da2) Co-authored-by: Seth Michael Larson Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Serhiy Storchaka Co-authored-by: Gregory P. Smith --- Lib/ipaddress.py | 11 +++++++++-- Lib/test/test_ipaddress.py | 11 +++++++++++ .../2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst | 2 ++ 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index d8a84f33264dc5..69e933a6541f78 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -1660,8 +1660,16 @@ def _ip_int_from_string(cls, ip_str): """ if not ip_str: raise AddressValueError('Address cannot be empty') + if len(ip_str) > 39: + msg = ("At most 39 characters expected in " + f"{ip_str[:14]!r}({len(ip_str)-28} chars elided){ip_str[-14:]!r}") + raise AddressValueError(msg) - parts = ip_str.split(':') + # We want to allow more parts than the max to be 'split' + # to preserve the correct error message when there are + # too many parts combined with '::' + _max_parts = cls._HEXTET_COUNT + 1 + parts = ip_str.split(':', maxsplit=_max_parts) # An IPv6 address needs at least 2 colons (3 parts). _min_parts = 3 @@ -1681,7 +1689,6 @@ def _ip_int_from_string(cls, ip_str): # An IPv6 address can't have more than 8 colons (9 parts). # The extra colon comes from using the "::" notation for a single # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 if len(parts) > _max_parts: msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) raise AddressValueError(msg) diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index a06608d0016253..ee95454e64bb0d 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -397,6 +397,17 @@ def assertBadSplit(addr): # A trailing IPv4 address is two parts assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42%scope") + def test_bad_address_split_v6_too_long(self): + def assertBadSplit(addr): + msg = r"At most 39 characters expected in %s" + with self.assertAddressError(msg, repr(re.escape(addr[:14]))): + ipaddress.IPv6Address(addr) + + # Long IPv6 address + long_addr = ("0:" * 10000) + "0" + assertBadSplit(long_addr) + assertBadSplit(long_addr + "%zoneid") + def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): msg = "Exactly 8 parts expected without '::' in %r" diff --git a/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst new file mode 100644 index 00000000000000..b57ec3e70dcc5f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst @@ -0,0 +1,2 @@ +Short-circuit the processing of long IPv6 addresses early in :mod:`ipaddress` to prevent excessive +memory consumption and a minor denial-of-service. From de9444ef94abe0cc8f72c26cd3900f4c66883646 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 05:34:17 +0200 Subject: [PATCH 225/713] [3.14] gh-134546: ensure remote pdb script is readable (GH-134552) (#134616) gh-134546: ensure remote pdb script is readable (GH-134552) (cherry picked from commit 74a9c60f3ee2195e506144c3811090f1334c996b) Co-authored-by: Anthony Sottile --- Lib/pdb.py | 3 +++ .../Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst | 1 + 2 files changed, 4 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst diff --git a/Lib/pdb.py b/Lib/pdb.py index d9e9bc011dbcca..c8c3134d3eb721 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -75,6 +75,7 @@ import code import glob import json +import stat import token import types import atexit @@ -3419,6 +3420,8 @@ def attach(pid, commands=()): ) ) connect_script.close() + orig_mode = os.stat(connect_script.name).st_mode + os.chmod(connect_script.name, orig_mode | stat.S_IROTH | stat.S_IRGRP) sys.remote_exec(pid, connect_script.name) # TODO Add a timeout? Or don't bother since the user can ^C? diff --git a/Misc/NEWS.d/next/Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst b/Misc/NEWS.d/next/Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst new file mode 100644 index 00000000000000..eea897f59181c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst @@ -0,0 +1 @@ +Ensure :mod:`pdb` remote debugging script is readable by remote Python process. From 00fd5440f03e55718bad76dbc45725f24744596c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 06:16:02 +0200 Subject: [PATCH 226/713] [3.14] gh-80334: fix multiprocessing.freeze_support for other spawn platforms (GH-134462) (#134619) gh-80334: fix multiprocessing.freeze_support for other spawn platforms (GH-134462) Doc/library/multiprocessing.rst: freeze_support: Change to specify spawn method instead of platform Have multiprocessing.freeze_support() enable on spawn, not just win32. --------- (cherry picked from commit 80284b5c5eebd0e603c38322f94a97a2853ceeba) Co-authored-by: Eddy Mulyono Co-authored-by: Gregory P. Smith --- Doc/library/multiprocessing.rst | 10 +++++----- Lib/multiprocessing/context.py | 2 +- .../2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6c43d5fe166e2f..6f37930ea80391 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1081,7 +1081,7 @@ Miscellaneous .. function:: freeze_support() Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, + frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) One needs to call this function straight after the ``if __name__ == @@ -1099,10 +1099,10 @@ Miscellaneous If the ``freeze_support()`` line is omitted then trying to run the frozen executable will raise :exc:`RuntimeError`. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. + Calling ``freeze_support()`` has no effect when the start method is not + *spawn*. In addition, if the module is being run normally by the Python + interpreter (the program has not been frozen), then ``freeze_support()`` + has no effect. .. function:: get_all_start_methods() diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index d0a3ad00e53ad8..051d567d457928 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -145,7 +145,7 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() diff --git a/Misc/NEWS.d/next/Library/2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst b/Misc/NEWS.d/next/Library/2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst new file mode 100644 index 00000000000000..228429516db416 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst @@ -0,0 +1,2 @@ +:func:`multiprocessing.freeze_support` now checks for work on any "spawn" +start method platform rather than only on Windows. From 162e3f35110a49084e8e9c0d1267864df8da43df Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 09:21:34 +0200 Subject: [PATCH 227/713] [3.14] gh-134595: Update HOWTO to reflect change in CIBW option (GH-134598) (#134622) Co-authored-by: Jasper Wong <111284156+3willows@users.noreply.github.com> --- Doc/howto/free-threading-extensions.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 5a3970f15d52d6..175bb5dc831848 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -396,7 +396,7 @@ The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. free-threaded build, with the ``t`` suffix, such as ``python3.13t``. * `pypa/cibuildwheel `_ supports the free-threaded build if you set - `CIBW_FREE_THREADED_SUPPORT `_. + `CIBW_ENABLE to cpython-freethreading `_. Limited C API and Stable ABI ............................ From 81f099375e3007709e3668e603edd188c05d662d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 24 May 2025 15:34:31 +0200 Subject: [PATCH 228/713] [3.14] gh-134168: fix `http.server` CLI support for IPv6 and `--directory` when serving over HTTPS (GH-134169) (#134630) [3.14] gh-134168: fix `http.server` CLI support for IPv6 and `--directory` when serving over HTTPS (GH-134169) (cherry picked from commit 2fd09b011031f3c00c342b44e02e2817010e507c) Co-authored-by: ggqlq <124190229+ggqlq@users.noreply.github.com> --- Lib/http/server.py | 15 +++++++++++---- ...2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst | 2 ++ 2 files changed, 13 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst diff --git a/Lib/http/server.py b/Lib/http/server.py index 31fe9cae781f0a..dda32644a28f90 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1320,8 +1320,8 @@ def test(HandlerClass=BaseHTTPRequestHandler, HandlerClass.protocol_version = protocol if tls_cert: - server = ThreadingHTTPSServer(addr, HandlerClass, certfile=tls_cert, - keyfile=tls_key, password=tls_password) + server = ServerClass(addr, HandlerClass, certfile=tls_cert, + keyfile=tls_key, password=tls_password) else: server = ServerClass(addr, HandlerClass) @@ -1387,7 +1387,7 @@ def test(HandlerClass=BaseHTTPRequestHandler, handler_class = SimpleHTTPRequestHandler # ensure dual-stack is not disabled; ref #38907 - class DualStackServer(ThreadingHTTPServer): + class DualStackServerMixin: def server_bind(self): # suppress exception when protocol is IPv4 @@ -1400,9 +1400,16 @@ def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, directory=args.directory) + class HTTPDualStackServer(DualStackServerMixin, ThreadingHTTPServer): + pass + class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): + pass + + ServerClass = HTTPSDualStackServer if args.tls_cert else HTTPDualStackServer + test( HandlerClass=handler_class, - ServerClass=DualStackServer, + ServerClass=ServerClass, port=args.port, bind=args.bind, protocol=args.protocol, diff --git a/Misc/NEWS.d/next/Library/2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst b/Misc/NEWS.d/next/Library/2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst new file mode 100644 index 00000000000000..5a0e20005db11e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst @@ -0,0 +1,2 @@ +:mod:`http.server`: Fix IPv6 address binding and +:option:`--directory ` handling when using HTTPS. From 80d70defc22ba15d45bb2dc199a12af7a114561f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 24 May 2025 17:06:54 +0200 Subject: [PATCH 229/713] [3.14] gh-134578: Mark more slow tests (GH-134579) (GH-134590) (cherry picked from commit 77eade39f972a4f3d8e9fec00288779f35ceee21) Co-authored-by: Serhiy Storchaka --- Lib/test/test_ast/test_ast.py | 1 + Lib/test/test_capi/test_object.py | 1 + Lib/test/test_collections.py | 2 ++ Lib/test/test_json/test_recursion.py | 1 + Lib/test/test_statistics.py | 1 + 5 files changed, 6 insertions(+) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 1479a8eafd62ec..46745cfa8f8325 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3292,6 +3292,7 @@ def check_output(self, source, expect, *flags): expect = self.text_normalize(expect) self.assertEqual(res, expect) + @support.requires_resource('cpu') def test_invocation(self): # test various combinations of parameters base_flags = ( diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 127862546b1bce..98a9a096cfee4d 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -221,6 +221,7 @@ def test_decref_freed_object(self): """ self.check_negative_refcount(code) + @support.requires_resource('cpu') def test_decref_delayed(self): # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path # handles destructors that are possibly re-entrant or trigger a GC. diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1e93530398be79..d9d61e5c2053e3 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -542,6 +542,8 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) + @support.requires_resource('cpu') + def test_large_size(self): n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 8f0e5e078ed0d4..5d7b56ff9ad285 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -86,6 +86,7 @@ def test_highly_nested_objects_decoding(self): @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 5980f939185965..0dd619dd7c8ceb 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -2346,6 +2346,7 @@ def test_mixed_int_and_float(self): class TestKDE(unittest.TestCase): + @support.requires_resource('cpu') def test_kde(self): kde = statistics.kde StatisticsError = statistics.StatisticsError From 93aee568c06b956aa32d5418a12090764c67d4d1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 15:54:28 +0200 Subject: [PATCH 230/713] [3.14] GH-130328: Speedup pasting in legacy console on Windows (gh-133728) (#134653) (cherry picked from commit 91b48868a82821c1cbfd2245212699b57d3a24c7) Co-authored-by: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Lib/_pyrepl/commands.py | 8 +++++- Lib/_pyrepl/windows_console.py | 26 +++++++++++-------- Lib/test/test_pyrepl/test_unix_console.py | 1 + Lib/test/test_pyrepl/test_windows_console.py | 1 + ...-05-09-09-10-34.gh-issue-130328.s9h4By.rst | 2 ++ 5 files changed, 26 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-09-09-10-34.gh-issue-130328.s9h4By.rst diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2354fbb2ec2c1e..50c824995d85b8 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -370,6 +370,13 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) + if r.paste_mode: + data = "" + ev = r.console.getpending() + data += ev.data + if data: + r.insert(data) + r.last_refresh_cache.invalidated = True class insert_nl(EditCommand): @@ -484,7 +491,6 @@ def do(self) -> None: data = "" start = time.time() while done not in data: - self.reader.console.wait(100) ev = self.reader.console.getpending() data += ev.data trace( diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 95749198b3b2f9..c56dcd6d7dd434 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -419,10 +419,7 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self, block: bool = True) -> INPUT_RECORD | None: - if not block and not self.wait(timeout=0): - return None - + def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): @@ -431,14 +428,10 @@ def _read_input(self, block: bool = True) -> INPUT_RECORD | None: return rec def _read_input_bulk( - self, block: bool, n: int + self, n: int ) -> tuple[ctypes.Array[INPUT_RECORD], int]: rec = (n * INPUT_RECORD)() read = DWORD() - - if not block and not self.wait(timeout=0): - return rec, 0 - if not ReadConsoleInput(InHandle, rec, n, read): raise WinError(GetLastError()) @@ -449,8 +442,11 @@ def get_event(self, block: bool = True) -> Event | None: and there is no event pending, otherwise waits for the completion of an event.""" + if not block and not self.wait(timeout=0): + return None + while self.event_queue.empty(): - rec = self._read_input(block) + rec = self._read_input() if rec is None: return None @@ -551,12 +547,20 @@ def getpending(self) -> Event: if e2: e.data += e2.data - recs, rec_count = self._read_input_bulk(False, 1024) + recs, rec_count = self._read_input_bulk(1024) for i in range(rec_count): rec = recs[i] + # In case of a legacy console, we do not only receive a keydown + # event, but also a keyup event - and for uppercase letters + # an additional SHIFT_PRESSED event. if rec and rec.EventType == KEY_EVENT: key_event = rec.Event.KeyEvent + if not key_event.bKeyDown: + continue ch = key_event.uChar.UnicodeChar + if ch == "\x00": + # ignore SHIFT_PRESSED and special keys + continue if ch == "\r": ch += "\n" e.data += ch diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index c447b310c49a06..b3f7dc028fe210 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -20,6 +20,7 @@ def unix_console(events, **kwargs): console = UnixConsole() console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) height = kwargs.get("height", 25) width = kwargs.get("width", 80) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index e7bab226b31ddf..a52ae96a83ddde 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -35,6 +35,7 @@ class WindowsConsoleTests(TestCase): def console(self, events, **kwargs) -> Console: console = WindowsConsole() console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) console.wait = MagicMock() console._scroll = MagicMock() console._hide_cursor = MagicMock() diff --git a/Misc/NEWS.d/next/Library/2025-05-09-09-10-34.gh-issue-130328.s9h4By.rst b/Misc/NEWS.d/next/Library/2025-05-09-09-10-34.gh-issue-130328.s9h4By.rst new file mode 100644 index 00000000000000..00b556c6a336ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-09-09-10-34.gh-issue-130328.s9h4By.rst @@ -0,0 +1,2 @@ +Speedup pasting in ``PyREPL`` on Windows in a legacy console. Patch by Chris +Eibl. From d82d445b18c24ca6b6afdb957a72636eab88efad Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 18:04:22 +0200 Subject: [PATCH 231/713] [3.14] gh-133778: Fix setting `__annotations__` under PEP 563 (GH-133794) (#134655) gh-133778: Fix setting `__annotations__` under PEP 563 (GH-133794) (cherry picked from commit 4443110c3409ecba9f0fd49495d039030cb7ed58) Co-authored-by: Jelle Zijlstra --- Lib/test/test_type_annotations.py | 22 +++++++++ ...-05-09-18-11-21.gh-issue-133778.pWEV3t.rst | 2 + Objects/typeobject.c | 45 +++++++++++++++---- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-09-18-11-21.gh-issue-133778.pWEV3t.rst diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 2c886bb6d362fa..c66cb058552643 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -498,6 +498,28 @@ def f(x: int) -> int: pass self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(f.__annotations__, annos) + def test_set_annotations(self): + function_code = textwrap.dedent(""" + def f(x: int): + pass + """) + class_code = textwrap.dedent(""" + class f: + x: int + """) + for future in (False, True): + for label, code in (("function", function_code), ("class", class_code)): + with self.subTest(future=future, label=label): + if future: + code = "from __future__ import annotations\n" + code + ns = run_code(code) + f = ns["f"] + anno = "int" if future else int + self.assertEqual(f.__annotations__, {"x": anno}) + + f.__annotations__ = {"x": str} + self.assertEqual(f.__annotations__, {"x": str}) + def test_name_clash_with_format(self): # this test would fail if __annotate__'s parameter was called "format" # during symbol table construction diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-09-18-11-21.gh-issue-133778.pWEV3t.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-09-18-11-21.gh-issue-133778.pWEV3t.rst new file mode 100644 index 00000000000000..6eb6881213c434 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-09-18-11-21.gh-issue-133778.pWEV3t.rst @@ -0,0 +1,2 @@ +Fix bug where assigning to the :attr:`~type.__annotations__` attributes of +classes defined under ``from __future__ import annotations`` had no effect. diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a7ab69fef4c721..ee09289425b91a 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2065,19 +2065,46 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } - int result; PyObject *dict = PyType_GetDict(type); - if (value != NULL) { - /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); - } else { - /* delete */ - result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); - if (result == 0) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + int result = PyDict_ContainsString(dict, "__annotations__"); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (result) { + // If __annotations__ is currently in the dict, we update it, + if (value != NULL) { + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); + } else { + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { + // Somebody else just deleted it? + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } + if (result < 0) { Py_DECREF(dict); return -1; } + // Also clear __annotations_cache__ just in case. + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + } + else { + // Else we update only __annotations_cache__. + if (value != NULL) { + /* set */ + result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); + } else { + /* delete */ + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + if (result == 0) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } } if (result < 0) { Py_DECREF(dict); From cbf4ccf1d02b39f8c2626d5af42a5e85b7d1f055 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 18:05:19 +0200 Subject: [PATCH 232/713] [3.14] gh-133684: Fix get_annotations() where PEP 563 is involved (GH-133795) (#134656) gh-133684: Fix get_annotations() where PEP 563 is involved (GH-133795) (cherry picked from commit 3e562b394252ff75d9809b7940020a775e4df68b) Co-authored-by: Jelle Zijlstra --- Lib/annotationlib.py | 19 +++++- Lib/test/test_annotationlib.py | 66 ++++++++++++++++++- ...-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst | 3 + 3 files changed, 84 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index 32b8553458930c..a7dfb91515a1c4 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -1042,14 +1042,27 @@ def _get_and_call_annotate(obj, format): return None +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + def _get_dunder_annotations(obj): """Return the annotations for an object, checking that it is a dictionary. Does not return a fresh dictionary. """ - ann = getattr(obj, "__annotations__", None) - if ann is None: - return None + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index 73a821d15e3481..fe091e52a86dc4 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,7 +7,7 @@ import functools import itertools import pickle -from string.templatelib import Interpolation, Template +from string.templatelib import Template import typing import unittest from annotationlib import ( @@ -815,6 +815,70 @@ def test_stringized_annotations_on_class(self): {"x": int}, ) + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + def test_modify_annotations(self): def f(x: int): pass diff --git a/Misc/NEWS.d/next/Library/2025-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst b/Misc/NEWS.d/next/Library/2025-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst new file mode 100644 index 00000000000000..0cb1bc237a12c9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst @@ -0,0 +1,3 @@ +Fix bug where :func:`annotationlib.get_annotations` would return the wrong +result for certain classes that are part of a class hierarchy where ``from +__future__ import annotations`` is used. From c61fd7c0f2df757fb485f60a8e6a5f8164477566 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 18:48:19 +0200 Subject: [PATCH 233/713] [3.14] gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (GH-134603) (#134659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-134582: Fix t-strings untokenize() roundtrip removing space between braces (GH-134603) (cherry picked from commit 52509cc94b1a18cb325dbfa7e5f830b32759a903) Co-authored-by: Loïc Simon --- Lib/test/test_tokenize.py | 4 ++++ Lib/tokenize.py | 14 +++++++------- .../2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst | 1 + 3 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 2d41a5e5ac0697..e6b19fe1812d44 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1975,6 +1975,10 @@ def test_roundtrip(self): for case in cases: self.check_roundtrip(case) + self.check_roundtrip(r"t'{ {}}'") + self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'") + self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'") + def test_continuation(self): # Balancing continuation diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 8d01fd7bce41b0..559a7aecbde2d1 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -274,7 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False - in_fstring = 0 + in_fstring_or_tstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -293,10 +293,10 @@ def compat(self, token, iterable): else: prevstring = False - if toknum == FSTRING_START: - in_fstring += 1 - elif toknum == FSTRING_END: - in_fstring -= 1 + if toknum in {FSTRING_START, TSTRING_START}: + in_fstring_or_tstring += 1 + elif toknum in {FSTRING_END, TSTRING_END}: + in_fstring_or_tstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -311,8 +311,8 @@ def compat(self, token, iterable): elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}: tokval = self.escape_brackets(tokval) - # Insert a space between two consecutive brackets if we are in an f-string - if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + # Insert a space between two consecutive brackets if we are in an f-string or t-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring: tokval = ' ' + tokval # Insert a space between two consecutive f-strings diff --git a/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst new file mode 100644 index 00000000000000..23e1d5891b685f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst @@ -0,0 +1 @@ +Fix tokenize.untokenize() round-trip errors related to t-strings braces escaping From 8c6653428f32189e4779a5ed444cf42e811368e0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 19:39:44 +0200 Subject: [PATCH 234/713] [3.14] annotationlib docs: note that ForwardRef.evaluate eventually defaults to empty globals (GH-134661) (#134662) annotationlib docs: note that ForwardRef.evaluate eventually defaults to empty globals (GH-134661) (cherry picked from commit b51b08a0a5fedde4f74e4cc338b8b5ad9656ad50) Co-authored-by: Jelle Zijlstra --- Doc/library/annotationlib.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 41c9ce479ff0f8..7dfc11449a6cbc 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -211,6 +211,10 @@ Classes means may not have any information about their scope, so passing arguments to this method may be necessary to evaluate them successfully. + If no *owner*, *globals*, *locals*, or *type_params* are provided and the + :class:`~ForwardRef` does not contain information about its origin, + empty globals and locals dictionaries are used. + .. versionadded:: 3.14 From 9c6d2dbe26e30114773bf55147d36960cb89e519 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 19:52:04 +0200 Subject: [PATCH 235/713] [3.14] gh-133960: Improve typing.evaluate_forward_ref (GH-133961) (#134663) gh-133960: Improve typing.evaluate_forward_ref (GH-133961) As explained in GH-133960, this removes most of the behavior differences with ForwardRef.evaluate. The remaining difference is about recursive evaluation of forwardrefs; this is practically useful in cases where an annotation refers to a type alias that itself is string-valued. This also improves several edge cases that were previously not handled optimally. For example, the function now takes advantage of the partial evaluation behavior of ForwardRef.evaluate() to evaluate more ForwardRefs in the FORWARDREF format. This also fixes GH-133959 as a side effect, because the buggy behavior in GH-133959 derives from evaluate_forward_ref(). (cherry picked from commit 57fef27cfc2bdfc1e3a65ef8c8a760198d15b14d) Co-authored-by: Jelle Zijlstra --- Doc/library/typing.rst | 15 +-- Lib/test/test_typing.py | 116 +++++++++++++++--- Lib/typing.py | 52 ++++---- ...-05-12-20-38-57.gh-issue-133960.Aee79f.rst | 3 + 4 files changed, 131 insertions(+), 55 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-12-20-38-57.gh-issue-133960.Aee79f.rst diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index 54cc3ea3311adf..dd8ea3c364f49a 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -3500,20 +3500,11 @@ Introspection helpers Evaluate an :class:`annotationlib.ForwardRef` as a :term:`type hint`. This is similar to calling :meth:`annotationlib.ForwardRef.evaluate`, - but unlike that method, :func:`!evaluate_forward_ref` also: - - * Recursively evaluates forward references nested within the type hint. - * Raises :exc:`TypeError` when it encounters certain objects that are - not valid type hints. - * Replaces type hints that evaluate to :const:`!None` with - :class:`types.NoneType`. - * Supports the :attr:`~annotationlib.Format.FORWARDREF` and - :attr:`~annotationlib.Format.STRING` formats. + but unlike that method, :func:`!evaluate_forward_ref` also + recursively evaluates forward references nested within the type hint. See the documentation for :meth:`annotationlib.ForwardRef.evaluate` for - the meaning of the *owner*, *globals*, *locals*, and *type_params* parameters. - *format* specifies the format of the annotation and is a member of - the :class:`annotationlib.Format` enum. + the meaning of the *owner*, *globals*, *locals*, *type_params*, and *format* parameters. .. versionadded:: 3.14 diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index b34bfcc90b1577..983fb39f4dbdcd 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -6859,12 +6859,10 @@ def test_forward_ref_and_final(self): self.assertEqual(hints, {'value': Final}) def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) + # This is not meaningful but we don't raise for it. + # https://github.com/python/cpython/issues/133959 + hints = get_type_hints(ann_module6) + self.assertEqual(hints, {'wrong': ClassVar[int]}) def test_get_type_hints_typeddict(self): self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) @@ -6967,6 +6965,11 @@ def foo(a: 'Callable[..., T]'): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': Callable[..., T]}) + def test_special_forms_no_forward(self): + def f(x: ClassVar[int]): + pass + self.assertEqual(get_type_hints(f), {'x': ClassVar[int]}) + def test_special_forms_forward(self): class C: @@ -6982,8 +6985,9 @@ class CF: self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) self.assertEqual(get_type_hints(C, globals())['y'], Final) - with self.assertRaises(TypeError): - get_type_hints(CF, globals()), + lfi = get_type_hints(CF, globals())['b'] + self.assertIs(get_origin(lfi), list) + self.assertEqual(get_args(lfi), (Final[int],)) def test_union_forward_recursion(self): ValueList = List['Value'] @@ -7216,33 +7220,113 @@ class C(Generic[T]): pass class EvaluateForwardRefTests(BaseTestCase): def test_evaluate_forward_ref(self): int_ref = ForwardRef('int') - missing = ForwardRef('missing') + self.assertIs(typing.evaluate_forward_ref(int_ref), int) self.assertIs( typing.evaluate_forward_ref(int_ref, type_params=()), int, ) + self.assertIs( + typing.evaluate_forward_ref(int_ref, format=annotationlib.Format.VALUE), + int, + ) self.assertIs( typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.FORWARDREF, + int_ref, format=annotationlib.Format.FORWARDREF, ), int, ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_undefined(self): + missing = ForwardRef('missing') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(missing) self.assertIs( typing.evaluate_forward_ref( - missing, type_params=(), format=annotationlib.Format.FORWARDREF, + missing, format=annotationlib.Format.FORWARDREF, ), missing, ) self.assertEqual( typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.STRING, + missing, format=annotationlib.Format.STRING, ), - 'int', + "missing", ) - def test_evaluate_forward_ref_no_type_params(self): - ref = ForwardRef('int') - self.assertIs(typing.evaluate_forward_ref(ref), int) + def test_evaluate_forward_ref_nested(self): + ref = ForwardRef("int | list['str']") + self.assertEqual( + typing.evaluate_forward_ref(ref), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "int | list['str']", + ) + + why = ForwardRef('"\'str\'"') + self.assertIs(typing.evaluate_forward_ref(why), str) + + def test_evaluate_forward_ref_none(self): + none_ref = ForwardRef('None') + self.assertIs(typing.evaluate_forward_ref(none_ref), None) + + def test_globals(self): + A = "str" + ref = ForwardRef('list[A]') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + self.assertEqual( + typing.evaluate_forward_ref(ref, globals={'A': A}), + list[str], + ) + + def test_owner(self): + ref = ForwardRef("A") + + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + # We default to the globals of `owner`, + # so it no longer raises `NameError` + self.assertIs( + typing.evaluate_forward_ref(ref, owner=Loop), A + ) + + def test_inherited_owner(self): + # owner passed to evaluate_forward_ref + ref = ForwardRef("list['A']") + self.assertEqual( + typing.evaluate_forward_ref(ref, owner=Loop), + list[A], + ) + + # owner set on the ForwardRef + ref = ForwardRef("list['A']", owner=Loop) + self.assertEqual( + typing.evaluate_forward_ref(ref), + list[A], + ) + + def test_partial_evaluation(self): + ref = ForwardRef("list[A]") + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + list[EqualToForwardRef('A')], + ) class CollectionsAbcTests(BaseTestCase): diff --git a/Lib/typing.py b/Lib/typing.py index 30d53cb46f8a18..87fd53802ca36d 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -956,12 +956,8 @@ def evaluate_forward_ref( """Evaluate a forward reference as a type hint. This is similar to calling the ForwardRef.evaluate() method, - but unlike that method, evaluate_forward_ref() also: - - * Recursively evaluates forward references nested within the type hint. - * Rejects certain objects that are not valid type hints. - * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *STRING* formats. + but unlike that method, evaluate_forward_ref() also + recursively evaluates forward references nested within the type hint. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -981,23 +977,24 @@ def evaluate_forward_ref( if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref - try: - value = forward_ref.evaluate(globals=globals, locals=locals, - type_params=type_params, owner=owner) - except NameError: - if format == _lazy_annotationlib.Format.FORWARDREF: - return forward_ref - else: - raise - - type_ = _type_check( - value, - "Forward references must evaluate to types.", - is_argument=forward_ref.__forward_is_argument__, - allow_special_forms=forward_ref.__forward_is_class__, - ) + if format is None: + format = _lazy_annotationlib.Format.VALUE + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner, format=format) + + if (isinstance(value, _lazy_annotationlib.ForwardRef) + and format == _lazy_annotationlib.Format.FORWARDREF): + return value + + if isinstance(value, str): + value = _make_forward_ref(value, module=forward_ref.__forward_module__, + owner=owner or forward_ref.__owner__, + is_argument=forward_ref.__forward_is_argument__, + is_class=forward_ref.__forward_is_class__) + if owner is None: + owner = forward_ref.__owner__ return _eval_type( - type_, + value, globals, locals, type_params, @@ -2338,12 +2335,12 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals for name, value in ann.items(): - if value is None: - value = type(None) if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) value = _eval_type(value, base_globals, base_locals, base.__type_params__, format=format, owner=obj) + if value is None: + value = type(None) hints[name] = value if include_extras or format == Format.STRING: return hints @@ -2377,8 +2374,6 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, localns = globalns type_params = getattr(obj, "__type_params__", ()) for name, value in hints.items(): - if value is None: - value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation @@ -2387,7 +2382,10 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) + value = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) + if value is None: + value = type(None) + hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} diff --git a/Misc/NEWS.d/next/Library/2025-05-12-20-38-57.gh-issue-133960.Aee79f.rst b/Misc/NEWS.d/next/Library/2025-05-12-20-38-57.gh-issue-133960.Aee79f.rst new file mode 100644 index 00000000000000..66e8483b25bc37 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-12-20-38-57.gh-issue-133960.Aee79f.rst @@ -0,0 +1,3 @@ +Simplify and improve :func:`typing.evaluate_forward_ref`. It now no longer +raises errors on certain invalid types. In several situations, it is now +able to evaluate forward references that were previously unsupported. From 6917fbf98f6fd7946ed92e7d6daa5a7ecb50167c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 20:43:07 +0200 Subject: [PATCH 236/713] [3.14] GH-130328: Fix WindowsConsoleGetEventTests after gh-133728 (gh-134660) (gh-134666) (cherry picked from commit 1000283694136ee0538baa6c6b2eee662ee618d4) Co-authored-by: Chris Eibl <138194463+chris-eibl@users.noreply.github.com> --- Lib/test/test_pyrepl/test_windows_console.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index a52ae96a83ddde..f9607e02c604ff 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -386,6 +386,7 @@ def get_event(self, input_records, **kwargs) -> Console: self.console._read_input = self.mock self.console._WindowsConsole__vt_support = kwargs.get("vt_support", False) + self.console.wait = MagicMock(return_value=True) event = self.console.get_event(block=False) return event From f5a8bfa335e48b8329d7ecf9dfa62fa06fdefa22 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 21:29:04 +0200 Subject: [PATCH 237/713] [3.14] Fix sphinx-lint warnings (default-role used). (GH-134647) (#134670) Fix sphinx-lint warnings (default-role used). (GH-134647) (cherry picked from commit 24a47155d2172966fab7d56f2bf9181056fba8d0) Co-authored-by: Julien Palard --- Doc/c-api/code.rst | 2 +- Doc/c-api/function.rst | 2 +- Doc/c-api/typeobj.rst | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6eae24b38fae48..42594f063b0709 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -182,7 +182,7 @@ bound into a function. Type of a code object watcher callback function. If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked - after `co` has been fully initialized. Otherwise, the callback is invoked + after *co* has been fully initialized. Otherwise, the callback is invoked before the destruction of *co* takes place, so the prior state of *co* can be inspected. diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 58792edeed25e3..63b78f677674e9 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -169,7 +169,7 @@ There are a few functions specific to Python functions. unpredictable effects, including infinite recursion. If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked - after `func` has been fully initialized. Otherwise, the callback is invoked + after *func* has been fully initialized. Otherwise, the callback is invoked before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 5df0c0fe608e53..91046c0e6f18ae 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1238,7 +1238,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a `~object.__dict__` + This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. From f68f05cbe3bdac0c668303ae55c6943b824d0c3f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 25 May 2025 23:16:10 +0200 Subject: [PATCH 238/713] [3.14] gh-115999: Add PyCodeObject.co_tlbc to the debug offsets (GH-134286) (#134348) gh-115999: Add PyCodeObject.co_tlbc to the debug offsets (GH-134286) (cherry picked from commit dd7f1130570d50461b2a0f81ab01c55b9ce93700) Signed-off-by: Pablo Galindo Co-authored-by: Pablo Galindo Salgado --- Doc/data/python3.14.abi | 62882 +++++++++++----------- Include/internal/pycore_debug_offsets.h | 8 + 2 files changed, 31452 insertions(+), 31438 deletions(-) diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index 64f4d473b72def..62e728532390be 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -1,31438 +1,31444 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 59d2c9d5377953..1a265c59ff8c08 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -52,9 +52,13 @@ extern "C" { #ifdef Py_GIL_DISABLED # define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) # define _Py_Debug_Free_Threaded 1 +# define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc) +# define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index) #else # define _Py_Debug_gilruntimestate_enabled 0 # define _Py_Debug_Free_Threaded 0 +# define _Py_Debug_code_object_co_tlbc 0 +# define _Py_Debug_interpreter_frame_tlbc_index 0 #endif @@ -109,6 +113,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplus; uint64_t owner; uint64_t stackpointer; + uint64_t tlbc_index; } interpreter_frame; // Code object offset; @@ -123,6 +128,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplusnames; uint64_t localspluskinds; uint64_t co_code_adaptive; + uint64_t co_tlbc; } code_object; // PyObject offset; @@ -265,6 +271,7 @@ typedef struct _Py_DebugOffsets { .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ + .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \ }, \ .code_object = { \ .size = sizeof(PyCodeObject), \ @@ -277,6 +284,7 @@ typedef struct _Py_DebugOffsets { .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + .co_tlbc = _Py_Debug_code_object_co_tlbc, \ }, \ .pyobject = { \ .size = sizeof(PyObject), \ From 1822f33b1af989ebb9e7a5b58bd40721fca9a218 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Sun, 25 May 2025 23:10:20 +0100 Subject: [PATCH 239/713] [3.14] gh-91048: Refactor and optimize remote debugging module (#134652) (#134673) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-91048: Refactor and optimize remote debugging module (#134652) Completely refactor Modules/_remote_debugging_module.c with improved code organization, replacing scattered reference counting and error handling with centralized goto error paths. This cleanup improves maintainability and reduces code duplication throughout the module while preserving the same external API. Implement memory page caching optimization in Python/remote_debug.h to avoid repeated reads of the same memory regions during debugging operations. The cache stores previously read memory pages and reuses them for subsequent reads, significantly reducing system calls and improving performance. Add code object caching mechanism with a new code_object_generation field in the interpreter state that tracks when code object caches need invalidation. This allows efficient reuse of parsed code object metadata and eliminates redundant processing of the same code objects across debugging sessions. Optimize memory operations by replacing multiple individual structure copies with single bulk reads for the same data structures. This reduces the number of memory operations and system calls required to gather debugging information from the target process. Update Makefile.pre.in to include Python/remote_debug.h in the headers list, ensuring that changes to the remote debugging header force proper recompilation of dependent modules and maintain build consistency across the codebase. Also, make the module compatible with the free threading build as an extra :) Co-authored-by: Łukasz Langa (cherry picked from commit 42b25ad4d3d6bcdc28ddfe07d2bf8831378bb0d1) --- Doc/data/python3.14.abi | 5882 +++++++++-------- Include/cpython/pystate.h | 2 + Include/internal/pycore_debug_offsets.h | 15 + .../pycore_global_objects_fini_generated.h | 1 + Include/internal/pycore_global_strings.h | 1 + Include/internal/pycore_interp_structs.h | 6 + .../internal/pycore_runtime_init_generated.h | 1 + .../internal/pycore_unicodeobject_generated.h | 4 + Lib/asyncio/tools.py | 8 +- Lib/test/test_external_inspection.py | 82 +- Makefile.pre.in | 1 + Modules/_remote_debugging_module.c | 2951 ++++++--- Modules/clinic/_remote_debugging_module.c.h | 243 + Objects/codeobject.c | 2 + Python/index_pool.c | 4 + Python/pystate.c | 10 +- Python/remote_debug.h | 117 +- 17 files changed, 5344 insertions(+), 3986 deletions(-) create mode 100644 Modules/clinic/_remote_debugging_module.c.h diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index 62e728532390be..375a1ee03e032d 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -1800,7 +1800,7 @@ - + @@ -6530,8 +6530,8 @@ - - + + @@ -10222,10 +10222,10 @@ - + - + @@ -15791,161 +15791,161 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -17434,373 +17434,390 @@ - + - + - + - + - + - + - - - - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + + + - + - + - + - + - - + + @@ -17808,12 +17825,12 @@ - + - + @@ -17822,8 +17839,8 @@ - - + + @@ -17840,21 +17857,21 @@ - + - + - + - + - + @@ -17863,7 +17880,7 @@ - + @@ -17883,7 +17900,7 @@ - + @@ -17923,7 +17940,7 @@ - + @@ -17985,7 +18002,7 @@ - + @@ -17999,104 +18016,104 @@ - + - + - + - + - + - + - + - - + + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -18104,7 +18121,7 @@ - + @@ -18112,7 +18129,7 @@ - + @@ -18120,7 +18137,7 @@ - + @@ -18128,7 +18145,7 @@ - + @@ -18136,7 +18153,7 @@ - + @@ -18144,7 +18161,7 @@ - + @@ -18152,7 +18169,7 @@ - + @@ -18160,7 +18177,7 @@ - + @@ -18168,7 +18185,7 @@ - + @@ -18176,7 +18193,7 @@ - + @@ -18184,7 +18201,7 @@ - + @@ -18192,2265 +18209,2268 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - + - + - + - - + + - - + + - - + + - + - + - + - - + + - - + + - + - + - + - + - - + + - + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - - + + - + - + - + - + - - + + - + - - + + - + - + - - + + - + - + - + - - + + - + - - + + - + - + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - + - - + + - - + + - + - + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - - + + - + - - + + - + - + - + - - + + - + - - + + - - + + - - + + - + - + - + - + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - - + + - + - - + + - + - - + + - + - + - + - - + + - + - + - + - + - - + + - + - - + + - + - + - + - - + + - + - + - - + + - + - - + + - + - + - - + + - + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - + - + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - + - + - + - + - - + + - + - + - - + + - + - + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - + - - + + - + - + - + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - + - + - - + + - - + + - - + + - + - + - + - - + + - - + + - + - - + + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - - + + - + - - + + - - + + - + - - + + - - + + - - + + - - + + - + - + - - + + - + - + - + - + - - + + - + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - - + + - - + + - - + + - + - - + + - + - + - - + + - - + + - - + + - - + + - - + + - + - + - - + + - + + + + - - + + - - + + - + @@ -20458,7 +20478,7 @@ - + @@ -20466,7 +20486,7 @@ - + @@ -20474,7 +20494,7 @@ - + @@ -20482,7 +20502,7 @@ - + @@ -20490,7 +20510,7 @@ - + @@ -20498,7 +20518,7 @@ - + @@ -20506,7 +20526,7 @@ - + @@ -20514,7 +20534,7 @@ - + @@ -20522,7 +20542,7 @@ - + @@ -20530,7 +20550,7 @@ - + @@ -20538,7 +20558,7 @@ - + @@ -20546,7 +20566,7 @@ - + @@ -20554,7 +20574,7 @@ - + @@ -20562,7 +20582,7 @@ - + @@ -20570,48 +20590,48 @@ - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + - + - + @@ -20620,22 +20640,22 @@ - - - - + + + + - - + + - + - - + + @@ -20643,10 +20663,10 @@ - + - + @@ -20664,19 +20684,19 @@ - + - - + + - - + + @@ -20684,37 +20704,37 @@ - - + + - + - + - + - + - + - + - + - + - + @@ -20723,7 +20743,7 @@ - + @@ -20749,19 +20769,19 @@ - + - - + + - + - + @@ -20773,20 +20793,20 @@ - + - + - + - + - + @@ -20795,10 +20815,10 @@ - + - + @@ -20807,7 +20827,7 @@ - + @@ -20815,17 +20835,17 @@ - + - + - + @@ -20836,7 +20856,7 @@ - + @@ -20844,19 +20864,19 @@ - + - - + + - + - + @@ -20864,10 +20884,10 @@ - + - + @@ -20887,7 +20907,7 @@ - + @@ -20934,21 +20954,21 @@ - + - + - + - + @@ -20956,7 +20976,7 @@ - + @@ -20979,13 +20999,13 @@ - + - + - + @@ -20996,12 +21016,12 @@ - + - + @@ -21015,7 +21035,7 @@ - + @@ -21029,7 +21049,7 @@ - + @@ -21038,7 +21058,7 @@ - + @@ -21055,8 +21075,8 @@ - - + + @@ -21076,7 +21096,7 @@ - + @@ -21090,7 +21110,7 @@ - + @@ -21106,7 +21126,7 @@ - + @@ -21114,15 +21134,15 @@ - + - + - + @@ -21146,12 +21166,12 @@ - + - + @@ -21174,19 +21194,19 @@ - - + + - + - + - + @@ -21195,7 +21215,7 @@ - + @@ -21203,7 +21223,7 @@ - + @@ -21214,7 +21234,7 @@ - + @@ -21225,7 +21245,7 @@ - + @@ -21234,7 +21254,7 @@ - + @@ -21242,10 +21262,10 @@ - + - + @@ -21257,8 +21277,8 @@ - - + + @@ -21266,12 +21286,12 @@ - + - + - + @@ -21285,7 +21305,7 @@ - + @@ -21293,18 +21313,18 @@ - + - + - + - + - + @@ -21318,8 +21338,8 @@ - - + + @@ -21354,285 +21374,288 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - + - + - + - + - + - - + + - + - + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - - - - - + + - - + + - + + + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - + - + - + - - - - + - - + + - + + + + - + - + - - + + - - + + - - + + - - + + - - + + - + + + + - + - + - + - + - + - + - + @@ -21676,21 +21699,21 @@ - + - + - + - + - - + + @@ -21701,14 +21724,14 @@ - - + + - - + + @@ -21716,71 +21739,71 @@ - + - + - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -21789,17 +21812,17 @@ - + - + - + @@ -21823,7 +21846,7 @@ - + @@ -21831,26 +21854,26 @@ - - + + - + - + - - + + - + @@ -21859,7 +21882,7 @@ - + @@ -21867,43 +21890,43 @@ - + - + - + - + - - + + - + - + - + - + - + - + - + @@ -21918,15 +21941,15 @@ - + - + - + @@ -21934,7 +21957,7 @@ - + @@ -21946,7 +21969,7 @@ - + @@ -21954,16 +21977,16 @@ - - + + - + - + @@ -21972,7 +21995,7 @@ - + @@ -21983,53 +22006,53 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22037,22 +22060,22 @@ - + - + - + - + - + @@ -22063,21 +22086,21 @@ - - - - - - - + - + - + + + + + + + @@ -22089,141 +22112,141 @@ - + - + - + - + - + - + - + - + - + - - - + + + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + @@ -22237,7 +22260,7 @@ - + @@ -22245,25 +22268,25 @@ - + - + - + - + - + @@ -22286,32 +22309,32 @@ - + - + - + - - + + - + - + @@ -22319,19 +22342,19 @@ - - + + - - + + - + @@ -22340,8 +22363,8 @@ - - + + @@ -22352,14 +22375,14 @@ - + - + @@ -22384,9 +22407,9 @@ - + - + @@ -22398,12 +22421,12 @@ - + - + - + @@ -22418,7 +22441,7 @@ - + @@ -22430,13 +22453,13 @@ - + - + - + @@ -22447,9 +22470,9 @@ - + - + @@ -22470,28 +22493,28 @@ - + - + - - - + + + - + - + - - - + + + @@ -22505,23 +22528,23 @@ - + - + - + - + - + @@ -22532,7 +22555,7 @@ - + @@ -22540,43 +22563,43 @@ - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + @@ -22611,36 +22634,36 @@ - - - + + + - + - - - - - + + + + + - - + + - + - + - + - + @@ -22648,8 +22671,8 @@ - - + + @@ -22657,11 +22680,11 @@ - + - + - + @@ -22670,10 +22693,10 @@ - - + + - + @@ -22682,10 +22705,10 @@ - - + + - + @@ -22697,21 +22720,21 @@ - + - + - + - - - - - - - + + + + + + + @@ -22734,24 +22757,24 @@ - + - + - + - + - - + + - + - + @@ -22769,29 +22792,29 @@ - - - - - - + + + + + + - + - - - + + + - - + + @@ -22799,9 +22822,9 @@ - - - + + + @@ -22815,36 +22838,36 @@ - + - + - + - + - + - + - + - + - + - + @@ -22852,7 +22875,7 @@ - + @@ -22860,10 +22883,10 @@ - + - + @@ -22871,10 +22894,10 @@ - + - + @@ -22891,7 +22914,7 @@ - + @@ -22899,18 +22922,18 @@ - + - + - + - + - + @@ -22918,7 +22941,7 @@ - + @@ -22926,7 +22949,7 @@ - + @@ -22937,8 +22960,8 @@ - - + + @@ -22949,9 +22972,9 @@ - - - + + + @@ -22989,10 +23012,10 @@ - + - + @@ -23001,7 +23024,7 @@ - + @@ -23013,19 +23036,19 @@ - + - + - + - + @@ -23040,22 +23063,22 @@ - - + + - + - + - + @@ -23081,19 +23104,19 @@ - + - + - + - + @@ -23141,11 +23164,11 @@ - + - - + + @@ -23228,7 +23251,7 @@ - + @@ -23267,10 +23290,10 @@ - + - + @@ -23291,7 +23314,7 @@ - + @@ -23302,11 +23325,11 @@ - + - - + + @@ -23329,14 +23352,14 @@ - + - - + + @@ -23344,10 +23367,10 @@ - - + + - + @@ -23356,7 +23379,7 @@ - + @@ -23364,8 +23387,8 @@ - - + + @@ -23379,13 +23402,13 @@ - - + + - + @@ -23400,7 +23423,7 @@ - + @@ -23436,7 +23459,7 @@ - + @@ -23451,171 +23474,171 @@ - + - - - - - - - - + + + + + + + + - + - - - + + + - - + + - - - - - - - - - - - + + + + + + + + + + - - - + + + + - + - - - - - - - + + + + + + + - - + + - + - - + + - - + + - - + + - - - - - + + + + + - - + + - - - - - + + + + + - - - + + + - + - + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + - + + + - - - - - - - - - - - - + + + + + + + + + + - - - + + + - - - + + + - - - - - + + + + + - + @@ -23924,272 +23947,272 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -24304,14 +24327,14 @@ - - + + - + - + @@ -24329,9 +24352,9 @@ - + - + @@ -24476,9 +24499,9 @@ - - - + + + @@ -24486,7 +24509,7 @@ - + @@ -24494,12 +24517,12 @@ - + - + @@ -24513,9 +24536,9 @@ - - - + + + @@ -24524,8 +24547,8 @@ - + @@ -24582,7 +24605,7 @@ - + @@ -24670,7 +24693,7 @@ - + @@ -24720,7 +24743,7 @@ - + @@ -24731,8 +24754,8 @@ - - + + @@ -24740,10 +24763,10 @@ - + - + @@ -24752,13 +24775,13 @@ - - + + - + @@ -24779,11 +24802,11 @@ - + - - + + @@ -24797,17 +24820,17 @@ - - - - - + + + + + - + - + @@ -24816,7 +24839,7 @@ - + @@ -24829,7 +24852,7 @@ - + @@ -24843,24 +24866,24 @@ - - + + - + - + - + @@ -24884,7 +24907,7 @@ - + @@ -24959,7 +24982,7 @@ - + @@ -24967,7 +24990,7 @@ - + @@ -25060,35 +25083,35 @@ - - + + - + - + - - + + - + - + - + - + - + - - + + @@ -25099,32 +25122,32 @@ - + - + - + - - - - - - - - - - - - + + + + + + + + + + + + - + @@ -25174,10 +25197,10 @@ - - - - + + + + @@ -25411,23 +25434,23 @@ - + - + - + + - - + @@ -25651,7 +25674,7 @@ - + @@ -25659,16 +25682,16 @@ - - - - - + + + + + - - - - + + + + @@ -25713,38 +25736,38 @@ - - + + - + - + - + - + - + - - + + - - + + @@ -25769,7 +25792,7 @@ - + @@ -25781,7 +25804,7 @@ - + @@ -25813,7 +25836,7 @@ - + @@ -25890,10 +25913,10 @@ - + - + @@ -25901,7 +25924,7 @@ - + @@ -25916,7 +25939,7 @@ - + @@ -25925,22 +25948,22 @@ - - + + - + - - + + - + - + - + @@ -25978,16 +26001,16 @@ - - + + - - + + @@ -26003,25 +26026,25 @@ - + - + - + - + - + - - + + @@ -26032,17 +26055,17 @@ - + - - + + - - + + @@ -26053,17 +26076,17 @@ - + - + - + - + @@ -26072,7 +26095,7 @@ - + @@ -26080,7 +26103,7 @@ - + @@ -26090,8 +26113,8 @@ - - + + @@ -26099,25 +26122,25 @@ - - + + - + - - + + - + - + @@ -26135,10 +26158,10 @@ - + - + @@ -26164,7 +26187,7 @@ - + @@ -26182,7 +26205,7 @@ - + @@ -26227,199 +26250,199 @@ - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + - - - + + + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - - - + + + + - - - + + + - + - + - + - + @@ -26429,15 +26452,15 @@ - - + + - - + + @@ -26447,71 +26470,71 @@ - + - + - + - - + + - + - + - - - - - - + + + + + + - + - + - + - + - + - + - - - - + + + + - + @@ -26532,10 +26555,10 @@ - + - + @@ -26547,24 +26570,24 @@ - + - + - - - - - + + + + + - + @@ -26572,33 +26595,33 @@ - + - + - + - - + + - + - + @@ -26607,24 +26630,24 @@ - - + + - - + + - + - + - + @@ -26634,17 +26657,17 @@ - - + + - - + + - + @@ -26659,7 +26682,7 @@ - + @@ -26668,14 +26691,14 @@ - + - + @@ -26687,11 +26710,11 @@ - + - + @@ -26705,50 +26728,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -26789,21 +26812,21 @@ - - - - - + + + + - - + + + - + - + @@ -26818,27 +26841,27 @@ - + - + - + - + - + @@ -26861,9 +26884,9 @@ - - - + + + @@ -26871,10 +26894,10 @@ - - + + - + @@ -26883,7 +26906,7 @@ - + @@ -26897,8 +26920,8 @@ - - + + @@ -26909,21 +26932,21 @@ - - + + - + - + - - - + + + @@ -26946,35 +26969,35 @@ - + - + - + - - - - - - - - - - - - + + + + + + + + + + + + - - + + @@ -26987,10 +27010,10 @@ - + - + @@ -27097,14 +27120,14 @@ - + - + - + @@ -27112,16 +27135,16 @@ - + - + - + @@ -27132,121 +27155,121 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + @@ -27282,21 +27305,21 @@ - + - + - + - + @@ -27304,27 +27327,27 @@ - - + + - + - + - + - + @@ -27462,15 +27485,15 @@ - + - + - + @@ -27478,11 +27501,11 @@ - - - - - + + + + + @@ -27526,7 +27549,7 @@ - + @@ -27597,7 +27620,7 @@ - + @@ -27671,9 +27694,9 @@ - + - + @@ -27713,17 +27736,17 @@ - + - + - - - - + + + + @@ -27744,7 +27767,7 @@ - + @@ -27758,7 +27781,7 @@ - + @@ -27766,15 +27789,15 @@ - - - - - - - - - + + + + + + + + + @@ -27834,7 +27857,7 @@ - + @@ -27869,15 +27892,15 @@ - + - + - + @@ -27967,8 +27990,8 @@ - - + + @@ -27980,11 +28003,11 @@ - - + + - + @@ -28002,7 +28025,7 @@ - + @@ -28015,7 +28038,7 @@ - + @@ -28024,22 +28047,22 @@ - + - - + + - - + + @@ -28056,7 +28079,7 @@ - + @@ -28065,7 +28088,7 @@ - + @@ -28073,24 +28096,24 @@ - + - + - + - + - + - + @@ -28100,9 +28123,9 @@ - - - + + + @@ -28110,48 +28133,48 @@ - + - - - + + + - + - + - + - + - + - + - + - - + + @@ -28182,7 +28205,7 @@ - + @@ -28246,7 +28269,7 @@ - + @@ -28319,7 +28342,7 @@ - + @@ -28331,11 +28354,11 @@ - + - - + + @@ -28355,8 +28378,8 @@ - - + + @@ -28364,7 +28387,7 @@ - + @@ -28376,27 +28399,27 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + - - + + - - + + - + @@ -28464,13 +28487,13 @@ - + - - + + @@ -28495,21 +28518,21 @@ - + - + - + - + @@ -28518,7 +28541,7 @@ - + @@ -28532,7 +28555,7 @@ - + @@ -28541,7 +28564,7 @@ - + @@ -28609,12 +28632,12 @@ - + - + @@ -28631,19 +28654,19 @@ - + - + - + @@ -28678,12 +28701,12 @@ - + - + @@ -28696,44 +28719,44 @@ - + - + - + - + - + - + - + - + - + @@ -28742,32 +28765,32 @@ - + - + - + - + - + @@ -28792,7 +28815,7 @@ - + @@ -28800,12 +28823,12 @@ - - + + - + @@ -28815,33 +28838,33 @@ - + - + - + - + - + @@ -28849,85 +28872,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -28936,62 +28959,62 @@ - + - + - + - + - + - + - + - + - - - + + + - + - - + + - + - - - - - + + + + + @@ -29003,8 +29026,8 @@ - - + + @@ -29050,12 +29073,12 @@ - + - + @@ -29063,8 +29086,8 @@ - - + + @@ -29073,53 +29096,53 @@ - + - + - + - + - + - - - + + + - + - - + + - + - - - - - - + + + + + + @@ -29135,7 +29158,7 @@ - + @@ -29144,21 +29167,21 @@ - + - + - + - + @@ -29174,65 +29197,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29240,18 +29263,18 @@ - + - + - + - + @@ -29269,13 +29292,13 @@ - + - + - + @@ -29287,16 +29310,16 @@ - + - + - - - - + + + + @@ -29398,21 +29421,21 @@ - + - - - - + + + + - + - + @@ -29421,27 +29444,27 @@ - + - + - + - + - + @@ -29507,16 +29530,16 @@ - - + + - + - + @@ -29530,7 +29553,7 @@ - + @@ -29542,23 +29565,23 @@ - - + + - + - + - + @@ -29576,11 +29599,11 @@ - + - + @@ -29590,36 +29613,36 @@ - + - + - + - + - - - - + + + + - + - + - + - + @@ -29631,25 +29654,25 @@ - - + + - + - + - + - + - + - + @@ -29657,7 +29680,7 @@ - + @@ -29665,59 +29688,59 @@ - + - + - + - + - - - - - + + + + + - + - + - + - + - + - - - + + + - - - - - - - + + + + + + + @@ -29887,13 +29910,13 @@ - + - - + + @@ -29925,7 +29948,7 @@ - + @@ -30165,7 +30188,7 @@ - + @@ -30273,7 +30296,7 @@ - + @@ -30282,8 +30305,8 @@ - - + + @@ -30348,19 +30371,19 @@ - + - - + + - - - - + + + + @@ -30380,7 +30403,7 @@ - + @@ -30442,7 +30465,7 @@ - + @@ -30453,7 +30476,7 @@ - + @@ -30466,117 +30489,117 @@ - + - + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - + - - + + - - + + - - - + + + - + - + - + - + - - + + - - - + + + @@ -30791,7 +30814,7 @@ - + @@ -30805,7 +30828,7 @@ - + @@ -30842,22 +30865,22 @@ - - - - + + + + - + - + - + @@ -30988,7 +31011,7 @@ - + @@ -30997,8 +31020,11 @@ - - + + + + + @@ -31019,7 +31045,7 @@ - + @@ -31114,10 +31140,10 @@ - - - - + + + + @@ -31147,7 +31173,7 @@ - + @@ -31168,8 +31194,8 @@ - - + + @@ -31191,26 +31217,26 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - - + + + @@ -31228,30 +31254,30 @@ - + - + - + - - + + - + - + @@ -31264,7 +31290,7 @@ - + @@ -31294,7 +31320,7 @@ - + @@ -31303,17 +31329,17 @@ - + - + - - + + @@ -31322,11 +31348,11 @@ - + - + @@ -31361,13 +31387,13 @@ - + - + @@ -31381,8 +31407,8 @@ - - + + diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7f1bc363861ddf..54d7e62292966e 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -61,6 +61,8 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; +/* Minimum size of data stack chunk */ +#define _PY_DATA_STACK_CHUNK_SIZE (16*1024) struct _ts { /* See Python/ceval.c for comments explaining most fields */ diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 1a265c59ff8c08..ce3fcb109f49f7 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -54,11 +54,13 @@ extern "C" { # define _Py_Debug_Free_Threaded 1 # define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc) # define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index) +# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) #else # define _Py_Debug_gilruntimestate_enabled 0 # define _Py_Debug_Free_Threaded 0 # define _Py_Debug_code_object_co_tlbc 0 # define _Py_Debug_interpreter_frame_tlbc_index 0 +# define _Py_Debug_interpreter_state_tlbc_generation 0 #endif @@ -89,6 +91,8 @@ typedef struct _Py_DebugOffsets { uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; + uint64_t code_object_generation; + uint64_t tlbc_generation; } interpreter_state; // Thread state offset; @@ -216,6 +220,11 @@ typedef struct _Py_DebugOffsets { uint64_t gi_frame_state; } gen_object; + struct _llist_node { + uint64_t next; + uint64_t prev; + } llist_node; + struct _debugger_support { uint64_t eval_breaker; uint64_t remote_debugger_support; @@ -251,6 +260,8 @@ typedef struct _Py_DebugOffsets { .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \ + .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \ }, \ .thread_state = { \ .size = sizeof(PyThreadState), \ @@ -347,6 +358,10 @@ typedef struct _Py_DebugOffsets { .gi_iframe = offsetof(PyGenObject, gi_iframe), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ + .llist_node = { \ + .next = offsetof(struct llist_node, next), \ + .prev = offsetof(struct llist_node, prev), \ + }, \ .debugger_support = { \ .eval_breaker = offsetof(PyThreadState, eval_breaker), \ .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support), \ diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 14fe9cdf0c77a0..6381bac38aa8d2 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -796,6 +796,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 197362a3e98be0..97547f771a6953 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -287,6 +287,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c3e6c77405bfe7..8a29c533b99058 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -726,6 +726,10 @@ typedef struct _PyIndexPool { // Next index to allocate if no free indices are available int32_t next_index; + + // Generation counter incremented on thread creation/destruction + // Used for TLBC cache invalidation in remote debugging + uint32_t tlbc_generation; } _PyIndexPool; typedef union _Py_unique_id_entry { @@ -843,6 +847,8 @@ struct _is { /* The per-interpreter GIL, which might not be used. */ struct _gil_runtime_state _gil; + uint64_t _code_object_generation; + /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index cb6e9c0b02ce2c..7143ad0a018eb5 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -794,6 +794,7 @@ extern "C" { INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(any), \ INIT_ID(append), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index dbb0f4e921149c..bdf0a0a2e33695 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -936,6 +936,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index b2da7d2f6ba10c..3fc4524c008db6 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,11 +1,10 @@ """Tools to analyze tasks running in asyncio programs.""" -from dataclasses import dataclass from collections import defaultdict from itertools import count from enum import Enum import sys -from _remote_debugging import get_all_awaited_by +from _remote_debugging import RemoteUnwinder class NodeType(Enum): @@ -118,6 +117,11 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + def build_async_tree(result, task_emoji="(T)", cor_emoji=""): """ Build a list of strings for pretty-print an async call tree. diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index ad3f669a03043e..291c419066ac5b 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4,6 +4,7 @@ import importlib import sys import socket +import threading from asyncio import staggered, taskgroups from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry @@ -16,9 +17,7 @@ try: from _remote_debugging import PROCESS_VM_READV_SUPPORTED - from _remote_debugging import get_stack_trace - from _remote_debugging import get_async_stack_trace - from _remote_debugging import get_all_awaited_by + from _remote_debugging import RemoteUnwinder except ImportError: raise unittest.SkipTest("Test only runs when _remote_debugging is available") @@ -34,7 +33,23 @@ def _make_test_script(script_dir, script_basename, source): ) +def get_stack_trace(pid): + unwinder = RemoteUnwinder(pid, all_threads=True) + return unwinder.get_stack_trace() + + +def get_async_stack_trace(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_async_stack_trace() + + +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + class TestGetStackTrace(unittest.TestCase): + maxDiff = None @skip_if_not_supported @unittest.skipIf( @@ -46,7 +61,7 @@ def test_remote_stack_trace(self): port = find_unused_port() script = textwrap.dedent( f"""\ - import time, sys, socket + import time, sys, socket, threading # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -55,13 +70,16 @@ def bar(): for x in range(100): if x == 50: baz() + def baz(): foo() def foo(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number - bar() + t = threading.Thread(target=bar) + t.start() + sock.sendall(b"ready:main\\n"); t.join() # same line number """ ) stack_trace = None @@ -82,8 +100,9 @@ def foo(): p = subprocess.Popen([sys.executable, script_name]) client_socket, _ = server_socket.accept() server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") + response = b"" + while b"ready:main" not in response or b"ready:thread" not in response: + response += client_socket.recv(1024) stack_trace = get_stack_trace(p.pid) except PermissionError: self.skipTest("Insufficient permissions to read the stack trace") @@ -94,13 +113,23 @@ def foo(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - expected_stack_trace = [ - ("foo", script_name, 14), - ("baz", script_name, 11), + thread_expected_stack_trace = [ + ("foo", script_name, 15), + ("baz", script_name, 12), ("bar", script_name, 9), - ("", script_name, 16), + ('Thread.run', threading.__file__, ANY) ] - self.assertEqual(stack_trace, expected_stack_trace) + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + self.assertIn((ANY, thread_expected_stack_trace), stack_trace) + + # Check that the main thread stack trace is in the result + frame = ("", script_name, 19) + for _, stack in stack_trace: + if frame in stack: + break + else: + self.fail("Main thread stack trace not found in result") @skip_if_not_supported @unittest.skipIf( @@ -700,13 +729,28 @@ async def main(): ) def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + this_tread_stack = None + for thread_id, stack in stack_trace: + if thread_id == threading.get_native_id(): + this_tread_stack = stack + break + self.assertIsNotNone(this_tread_stack) self.assertEqual( - stack_trace[0], - ( - "TestGetStackTrace.test_self_trace", - __file__, - self.test_self_trace.__code__.co_firstlineno + 6, - ), + stack[:2], + [ + ( + "get_stack_trace", + __file__, + get_stack_trace.__code__.co_firstlineno + 2, + ), + ( + "TestGetStackTrace.test_self_trace", + __file__, + self.test_self_trace.__code__.co_firstlineno + 6, + ), + ] ) diff --git a/Makefile.pre.in b/Makefile.pre.in index 3ab7c3d6c48ad9..b5703fbe6ae974 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1206,6 +1206,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ $(srcdir)/Include/weakrefobject.h \ + $(srcdir)/Python/remote_debug.h \ \ pyconfig.h \ $(PARSER_HEADERS) \ diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index f9f77a0814b4ce..004169e7a9c6a5 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -1,5 +1,16 @@ + /****************************************************************************** + * Python Remote Debugging Module + * + * This module provides functionality to debug Python processes remotely by + * reading their memory and reconstructing stack traces and asyncio task states. + ******************************************************************************/ + #define _GNU_SOURCE +/* ============================================================================ + * HEADERS AND INCLUDES + * ============================================================================ */ + #include #include #include @@ -23,6 +34,47 @@ # define HAVE_PROCESS_VM_READV 0 #endif +/* ============================================================================ + * TYPE DEFINITIONS AND STRUCTURES + * ============================================================================ */ + +#define GET_MEMBER(type, obj, offset) (*(type*)((char*)(obj) + (offset))) + +/* Size macros for opaque buffers */ +#define SIZEOF_BYTES_OBJ sizeof(PyBytesObject) +#define SIZEOF_CODE_OBJ sizeof(PyCodeObject) +#define SIZEOF_GEN_OBJ sizeof(PyGenObject) +#define SIZEOF_INTERP_FRAME sizeof(_PyInterpreterFrame) +#define SIZEOF_LLIST_NODE sizeof(struct llist_node) +#define SIZEOF_PAGE_CACHE_ENTRY sizeof(page_cache_entry_t) +#define SIZEOF_PYOBJECT sizeof(PyObject) +#define SIZEOF_SET_OBJ sizeof(PySetObject) +#define SIZEOF_TASK_OBJ 4096 +#define SIZEOF_THREAD_STATE sizeof(PyThreadState) +#define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) +#define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) +#define SIZEOF_LONG_OBJ sizeof(PyLongObject) + +// Calculate the minimum buffer size needed to read interpreter state fields +// We need to read code_object_generation and potentially tlbc_generation +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +#ifdef Py_GIL_DISABLED +#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)) +#else +#define INTERP_STATE_MIN_SIZE MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)) +#endif +#define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) + + + +// Copied from Modules/_asynciomodule.c because it's not exported + struct _Py_AsyncioModuleDebugOffsets { struct _asyncio_task_object { uint64_t size; @@ -45,6 +97,127 @@ struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; }; +typedef struct { + PyObject_HEAD + proc_handle_t handle; + uintptr_t runtime_start_address; + struct _Py_DebugOffsets debug_offsets; + int async_debug_offsets_available; + struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; + uintptr_t interpreter_addr; + uintptr_t tstate_addr; + uint64_t code_object_generation; + _Py_hashtable_t *code_object_cache; +#ifdef Py_GIL_DISABLED + // TLBC cache invalidation tracking + uint32_t tlbc_generation; // Track TLBC index pool changes + _Py_hashtable_t *tlbc_cache; // Cache of TLBC arrays by code object address +#endif +} RemoteUnwinderObject; + +typedef struct { + PyObject *func_name; + PyObject *file_name; + int first_lineno; + PyObject *linetable; // bytes + uintptr_t addr_code_adaptive; +} CachedCodeMetadata; + +typedef struct { + /* Types */ + PyTypeObject *RemoteDebugging_Type; +} RemoteDebuggingState; + +typedef struct +{ + int lineno; + int end_lineno; + int column; + int end_column; +} LocationInfo; + +typedef struct { + uintptr_t remote_addr; + size_t size; + void *local_copy; +} StackChunkInfo; + +typedef struct { + StackChunkInfo *chunks; + size_t count; +} StackChunkList; + +#include "clinic/_remote_debugging_module.c.h" + +/*[clinic input] +module _remote_debugging +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=5f507d5b2e76a7f7]*/ + + +/* ============================================================================ + * FORWARD DECLARATIONS + * ============================================================================ */ + +static int +parse_tasks_in_set( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + PyObject *awaited_by, + int recurse_task +); + +static int +parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to, + int recurse_task +); + +static int +parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +); + +/* Forward declarations for task parsing functions */ +static int parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* previous_frame +); + +/* ============================================================================ + * UTILITY FUNCTIONS AND HELPERS + * ============================================================================ */ + +static void +cached_code_metadata_destroy(void *ptr) +{ + CachedCodeMetadata *meta = (CachedCodeMetadata *)ptr; + Py_DECREF(meta->func_name); + Py_DECREF(meta->file_name); + Py_DECREF(meta->linetable); + PyMem_RawFree(meta); +} + +static inline RemoteDebuggingState * +RemoteDebugging_GetState(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (RemoteDebuggingState *)state; +} + +static inline int +RemoteDebugging_InitState(RemoteDebuggingState *st) +{ + return 0; +} + // Helper to chain exceptions and avoid repetitions static void chain_exceptions(PyObject *exception, const char *string) @@ -54,36 +227,14 @@ chain_exceptions(PyObject *exception, const char *string) _PyErr_ChainExceptions1(exc); } -// Get the PyAsyncioDebug section address for any platform -static uintptr_t -_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) -{ - uintptr_t address; - -#ifdef MS_WINDOWS - // On Windows, search for asyncio debug in executable or DLL - address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); -#elif defined(__linux__) - // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); -#elif defined(__APPLE__) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - if (address == 0) { - PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - } -#else - Py_UNREACHABLE(); -#endif - - return address; -} +/* ============================================================================ + * MEMORY READING FUNCTIONS + * ============================================================================ */ static inline int read_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) { - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void*), ptr_addr); + int result = _Py_RemoteDebug_PagedReadRemoteMemory(handle, address, sizeof(void*), ptr_addr); if (result < 0) { return -1; } @@ -93,7 +244,7 @@ read_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) static inline int read_Py_ssize_t(proc_handle_t *handle, uintptr_t address, Py_ssize_t *size) { - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(Py_ssize_t), size); + int result = _Py_RemoteDebug_PagedReadRemoteMemory(handle, address, sizeof(Py_ssize_t), size); if (result < 0) { return -1; } @@ -113,72 +264,53 @@ read_py_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) static int read_char(proc_handle_t *handle, uintptr_t address, char *result) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(char), result); - if (res < 0) { - return -1; - } - return 0; -} - -static int -read_sized_int(proc_handle_t *handle, uintptr_t address, void *result, size_t size) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, size, result); - if (res < 0) { - return -1; - } - return 0; -} - -static int -read_unsigned_long(proc_handle_t *handle, uintptr_t address, unsigned long *result) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(unsigned long), result); + int res = _Py_RemoteDebug_PagedReadRemoteMemory(handle, address, sizeof(char), result); if (res < 0) { return -1; } return 0; } -static int -read_pyobj(proc_handle_t *handle, uintptr_t address, PyObject *ptr_addr) -{ - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(PyObject), ptr_addr); - if (res < 0) { - return -1; - } - return 0; -} +/* ============================================================================ + * PYTHON OBJECT READING FUNCTIONS + * ============================================================================ */ static PyObject * read_py_str( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, + RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t max_len ) { PyObject *result = NULL; char *buf = NULL; - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->unicode_object.length, - sizeof(Py_ssize_t), - &len + // Read the entire PyUnicodeObject at once + char unicode_obj[SIZEOF_UNICODE_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_UNICODE_OBJ, + unicode_obj ); if (res < 0) { goto err; } + Py_ssize_t len = GET_MEMBER(Py_ssize_t, unicode_obj, unwinder->debug_offsets.unicode_object.length); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid string length (%zd) at 0x%lx", len, address); + return NULL; + } + buf = (char *)PyMem_RawMalloc(len+1); if (buf == NULL) { PyErr_NoMemory(); return NULL; } - size_t offset = debug_offsets->unicode_object.asciiobject_size; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); + size_t offset = unwinder->debug_offsets.unicode_object.asciiobject_size; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { goto err; } @@ -202,32 +334,40 @@ read_py_str( static PyObject * read_py_bytes( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, - uintptr_t address + RemoteUnwinderObject *unwinder, + uintptr_t address, + Py_ssize_t max_len ) { PyObject *result = NULL; char *buf = NULL; - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->bytes_object.ob_size, - sizeof(Py_ssize_t), - &len + // Read the entire PyBytesObject at once + char bytes_obj[SIZEOF_BYTES_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_BYTES_OBJ, + bytes_obj ); if (res < 0) { goto err; } + Py_ssize_t len = GET_MEMBER(Py_ssize_t, bytes_obj, unwinder->debug_offsets.bytes_object.ob_size); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid string length (%zd) at 0x%lx", len, address); + return NULL; + } + buf = (char *)PyMem_RawMalloc(len+1); if (buf == NULL) { PyErr_NoMemory(); return NULL; } - size_t offset = debug_offsets->bytes_object.ob_sval; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); + size_t offset = unwinder->debug_offsets.bytes_object.ob_sval; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); if (res < 0) { goto err; } @@ -249,45 +389,60 @@ read_py_bytes( return NULL; } - - static long -read_py_long(proc_handle_t *handle, _Py_DebugOffsets* offsets, uintptr_t address) +read_py_long( + RemoteUnwinderObject *unwinder, + uintptr_t address +) { unsigned int shift = PYLONG_BITS_IN_DIGIT; - Py_ssize_t size; - uintptr_t lv_tag; - - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->long_object.lv_tag, - sizeof(uintptr_t), - &lv_tag); + // Read the entire PyLongObject at once + char long_obj[SIZEOF_LONG_OBJ]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + unwinder->debug_offsets.long_object.size, + long_obj); if (bytes_read < 0) { return -1; } + uintptr_t lv_tag = GET_MEMBER(uintptr_t, long_obj, unwinder->debug_offsets.long_object.lv_tag); int negative = (lv_tag & 3) == 2; - size = lv_tag >> 3; + Py_ssize_t size = lv_tag >> 3; if (size == 0) { return 0; } - digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - return -1; - } + // If the long object has inline digits, use them directly + digit *digits; + if (size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { + // For small integers, digits are inline in the long_value.ob_digit array + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + return -1; + } + memcpy(digits, long_obj + unwinder->debug_offsets.long_object.ob_digit, size * sizeof(digit)); + } else { + // For larger integers, we need to read the digits separately + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + return -1; + } - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->long_object.ob_digit, - sizeof(digit) * size, - digits - ); - if (bytes_read < 0) { - goto error; + bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address + unwinder->debug_offsets.long_object.ob_digit, + sizeof(digit) * size, + digits + ); + if (bytes_read < 0) { + goto error; + } } long long value = 0; @@ -310,44 +465,115 @@ read_py_long(proc_handle_t *handle, _Py_DebugOffsets* offsets, uintptr_t address return -1; } +/* ============================================================================ + * ASYNCIO DEBUG FUNCTIONS + * ============================================================================ */ + +// Get the PyAsyncioDebug section address for any platform +static uintptr_t +_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) +{ + uintptr_t address; + +#ifdef MS_WINDOWS + // On Windows, search for asyncio debug in executable or DLL + address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__linux__) + // On Linux, search for asyncio debug in executable or DLL + address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__APPLE__) && TARGET_OS_OSX + // On macOS, try libpython first, then fall back to python + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + if (address == 0) { + PyErr_Clear(); + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); + } + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#else + Py_UNREACHABLE(); +#endif + + return address; +} + +static int +read_async_debug( + RemoteUnwinderObject *unwinder +) { + uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(&unwinder->handle); + if (!async_debug_addr) { + return -1; + } + + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, async_debug_addr, size, &unwinder->async_debug_offsets); + return result; +} + +/* ============================================================================ + * ASYNCIO TASK PARSING FUNCTIONS + * ============================================================================ */ + static PyObject * parse_task_name( - proc_handle_t *handle, - _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, + RemoteUnwinderObject *unwinder, uintptr_t task_address ) { - uintptr_t task_name_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_name, - &task_name_addr); - if (err) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + task_address, + unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj); + if (err < 0) { return NULL; } - // The task name can be a long or a string so we need to check the type + uintptr_t task_name_addr = GET_MEMBER(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_name); + task_name_addr &= ~Py_TAG_BITS; - PyObject task_name_obj; - err = read_pyobj( - handle, + // The task name can be a long or a string so we need to check the type + char task_name_obj[SIZEOF_PYOBJECT]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, task_name_addr, - &task_name_obj); - if (err) { + SIZEOF_PYOBJECT, + task_name_obj); + if (err < 0) { return NULL; } - unsigned long flags; - err = read_unsigned_long( - handle, - (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, - &flags); - if (err) { + // Now read the type object to get the flags + char type_obj[SIZEOF_TYPE_OBJ]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + GET_MEMBER(uintptr_t, task_name_obj, unwinder->debug_offsets.pyobject.ob_type), + SIZEOF_TYPE_OBJ, + type_obj); + if (err < 0) { return NULL; } - if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { - long res = read_py_long(handle, offsets, task_name_addr); + if ((GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(unwinder, task_name_addr); if (res == -1) { chain_exceptions(PyExc_RuntimeError, "Failed to get task name"); return NULL; @@ -355,355 +581,375 @@ parse_task_name( return PyUnicode_FromFormat("Task-%d", res); } - if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { + if(!(GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_UNICODE_SUBCLASS)) { PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); return NULL; } return read_py_str( - handle, - offsets, + unwinder, task_name_addr, 255 ); } -static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame -); - -static int -parse_coro_chain( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t coro_address, - PyObject *render_to +static int parse_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *awaited_by, + int recurse_task ) { - assert((void*)coro_address != NULL); - - uintptr_t gen_type_addr; - int err = read_ptr( - handle, - coro_address + offsets->pyobject.ob_type, - &gen_type_addr); - if (err) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { return -1; } - PyObject* name = NULL; - uintptr_t prev_frame; - if (parse_frame_object( - handle, - &name, - offsets, - coro_address + offsets->gen_object.gi_iframe, - &prev_frame) - < 0) - { - return -1; - } + uintptr_t task_ab_addr = GET_MEMBER(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); + task_ab_addr &= ~Py_TAG_BITS; - if (PyList_Append(render_to, name)) { - Py_DECREF(name); - return -1; + if ((void*)task_ab_addr == NULL) { + return 0; } - Py_DECREF(name); - int8_t gi_frame_state; - err = read_sized_int( - handle, - coro_address + offsets->gen_object.gi_frame_state, - &gi_frame_state, - sizeof(int8_t) - ); - if (err) { - return -1; - } + char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); - if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - char owner; - err = read_char( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.owner, - &owner - ); - if (err) { + if (awaited_by_is_a_set) { + if (parse_tasks_in_set(unwinder, task_ab_addr, awaited_by, recurse_task)) { return -1; } - if (owner != FRAME_OWNED_BY_GENERATOR) { - PyErr_SetString( - PyExc_RuntimeError, - "generator doesn't own its frame \\_o_/"); + } else { + if (parse_task(unwinder, task_ab_addr, awaited_by, recurse_task)) { return -1; } + } + + return 0; +} + +static int +handle_yield_from_frame( + RemoteUnwinderObject *unwinder, + uintptr_t gi_iframe_addr, + uintptr_t gen_type_addr, + PyObject *render_to +) { + // Read the entire interpreter frame at once + char iframe[SIZEOF_INTERP_FRAME]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + gi_iframe_addr, + SIZEOF_INTERP_FRAME, + iframe); + if (err < 0) { + return -1; + } + + if (GET_MEMBER(char, iframe, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + return -1; + } + + uintptr_t stackpointer_addr = GET_MEMBER(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer); + stackpointer_addr &= ~Py_TAG_BITS; - uintptr_t stackpointer_addr; + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; err = read_py_ptr( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.stackpointer, - &stackpointer_addr); + &unwinder->handle, + stackpointer_addr - sizeof(void*), + &gi_await_addr); if (err) { return -1; } - if ((void*)stackpointer_addr != NULL) { - uintptr_t gi_await_addr; - err = read_py_ptr( - handle, - stackpointer_addr - sizeof(void*), - &gi_await_addr); + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + err = read_ptr( + &unwinder->handle, + gi_await_addr + unwinder->debug_offsets.pyobject.ob_type, + &gi_await_addr_type_addr); if (err) { return -1; } - if ((void*)gi_await_addr != NULL) { - uintptr_t gi_await_addr_type_addr; - int err = read_ptr( - handle, - gi_await_addr + offsets->pyobject.ob_type, - &gi_await_addr_type_addr); + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain(unwinder, gi_await_addr, render_to); if (err) { return -1; } - - if (gen_type_addr == gi_await_addr_type_addr) { - /* This needs an explanation. We always start with parsing - native coroutine / generator frames. Ultimately they - are awaiting on something. That something can be - a native coroutine frame or... an iterator. - If it's the latter -- we can't continue building - our chain. So the condition to bail out of this is - to do that when the type of the current coroutine - doesn't match the type of whatever it points to - in its cr_await. - */ - err = parse_coro_chain( - handle, - offsets, - async_offsets, - gi_await_addr, - render_to - ); - if (err) { - return -1; - } - } } } - } return 0; } - static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task -); +parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + // Read the entire generator object at once + char gen_object[SIZEOF_GEN_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + coro_address, + SIZEOF_GEN_OBJ, + gen_object); + if (err < 0) { + return -1; + } + uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); -static int -parse_task( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, + PyObject* name = NULL; + + // Parse the previous frame using the gi_iframe from local copy + uintptr_t prev_frame; + uintptr_t gi_iframe_addr = coro_address + unwinder->debug_offsets.gen_object.gi_iframe; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &prev_frame) < 0) { + return -1; + } + + if (PyList_Append(render_to, name)) { + Py_DECREF(name); + return -1; + } + Py_DECREF(name); + + if (GET_MEMBER(int8_t, gen_object, unwinder->debug_offsets.gen_object.gi_frame_state) == FRAME_SUSPENDED_YIELD_FROM) { + return handle_yield_from_frame(unwinder, gi_iframe_addr, gen_type_addr, render_to); + } + + return 0; +} + +static PyObject* +create_task_result( + RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *render_to, int recurse_task ) { - char is_task; - int err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_is_task, - &is_task); - if (err) { - return -1; - } + PyObject* result = NULL; + PyObject *call_stack = NULL; + PyObject *tn = NULL; + char task_obj[SIZEOF_TASK_OBJ]; + uintptr_t coro_addr; - PyObject* result = PyList_New(0); + result = PyList_New(0); if (result == NULL) { - return -1; + goto error; } - PyObject *call_stack = PyList_New(0); + call_stack = PyList_New(0); if (call_stack == NULL) { - goto err; + goto error; } + if (PyList_Append(result, call_stack)) { - Py_DECREF(call_stack); - goto err; + goto error; } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(call_stack); + Py_CLEAR(call_stack); - if (is_task) { - PyObject *tn = NULL; - if (recurse_task) { - tn = parse_task_name( - handle, offsets, async_offsets, task_address); - } else { - tn = PyLong_FromUnsignedLongLong(task_address); + if (recurse_task) { + tn = parse_task_name(unwinder, task_address); + } else { + tn = PyLong_FromUnsignedLongLong(task_address); + } + if (tn == NULL) { + goto error; + } + + if (PyList_Append(result, tn)) { + goto error; + } + Py_CLEAR(tn); + + // Parse coroutine chain + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + goto error; + } + + coro_addr = GET_MEMBER(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_coro); + coro_addr &= ~Py_TAG_BITS; + + if ((void*)coro_addr != NULL) { + call_stack = PyList_New(0); + if (call_stack == NULL) { + goto error; } - if (tn == NULL) { - goto err; + + if (parse_coro_chain(unwinder, coro_addr, call_stack) < 0) { + Py_DECREF(call_stack); + goto error; } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto err; + + if (PyList_Reverse(call_stack)) { + Py_DECREF(call_stack); + goto error; } - Py_DECREF(tn); - uintptr_t coro_addr; - err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_coro, - &coro_addr); - if (err) { - goto err; + if (PyList_SetItem(result, 0, call_stack) < 0) { + Py_DECREF(call_stack); + goto error; } + } - if ((void*)coro_addr != NULL) { - err = parse_coro_chain( - handle, - offsets, - async_offsets, - coro_addr, - call_stack - ); - if (err) { - goto err; - } + return result; - if (PyList_Reverse(call_stack)) { - goto err; - } +error: + Py_XDECREF(result); + Py_XDECREF(call_stack); + Py_XDECREF(tn); + return NULL; +} + +static int +parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to, + int recurse_task +) { + char is_task; + PyObject* result = NULL; + PyObject* awaited_by = NULL; + int err; + + err = read_char( + &unwinder->handle, + task_address + unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task); + if (err) { + goto error; + } + + if (is_task) { + result = create_task_result(unwinder, task_address, recurse_task); + if (!result) { + goto error; + } + } else { + result = PyList_New(0); + if (result == NULL) { + goto error; } } if (PyList_Append(render_to, result)) { - goto err; + goto error; } if (recurse_task) { - PyObject *awaited_by = PyList_New(0); + awaited_by = PyList_New(0); if (awaited_by == NULL) { - goto err; + goto error; } + if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto err; + goto error; } - /* we can operate on a borrowed one to simplify cleanup */ Py_DECREF(awaited_by); - if (parse_task_awaited_by(handle, offsets, async_offsets, - task_address, awaited_by, 1) - ) { - goto err; + /* awaited_by is borrowed from 'result' to simplify cleanup */ + if (parse_task_awaited_by(unwinder, task_address, awaited_by, 1) < 0) { + // Clear the pointer so the cleanup doesn't try to decref it since + // it's borrowed from 'result' and will be decrefed when result is + // deleted. + awaited_by = NULL; + goto error; } } Py_DECREF(result); return 0; -err: - Py_DECREF(result); +error: + Py_XDECREF(result); + Py_XDECREF(awaited_by); return -1; } static int -parse_tasks_in_set( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t set_addr, +process_set_entry( + RemoteUnwinderObject *unwinder, + uintptr_t table_ptr, PyObject *awaited_by, int recurse_task ) { - uintptr_t set_obj; - if (read_py_ptr( - handle, - set_addr, - &set_obj) - ) { + uintptr_t key_addr; + if (read_py_ptr(&unwinder->handle, table_ptr, &key_addr)) { return -1; } - Py_ssize_t num_els; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.used, - &num_els) - ) { - return -1; - } + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_Py_ssize_t(&unwinder->handle, table_ptr, &ref_cnt)) { + return -1; + } - Py_ssize_t set_len; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.mask, - &set_len) - ) { - return -1; + if (ref_cnt) { + // if 'ref_cnt=0' it's a set dummy marker + if (parse_task(unwinder, key_addr, awaited_by, recurse_task)) { + return -1; + } + return 1; // Successfully processed a valid entry + } } - set_len++; // The set contains the `mask+1` element slots. + return 0; // Entry was NULL or dummy marker +} - uintptr_t table_ptr; - if (read_ptr( - handle, - set_obj + offsets->set_object.table, - &table_ptr) - ) { +static int +parse_tasks_in_set( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + PyObject *awaited_by, + int recurse_task +) { + char set_object[SIZEOF_SET_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + set_addr, + SIZEOF_SET_OBJ, + set_object); + if (err < 0) { return -1; } + Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); + Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; // The set contains the `mask+1` element slots. + uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + Py_ssize_t i = 0; Py_ssize_t els = 0; - while (i < set_len) { - uintptr_t key_addr; - if (read_py_ptr(handle, table_ptr, &key_addr)) { + while (i < set_len && els < num_els) { + int result = process_set_entry(unwinder, table_ptr, awaited_by, recurse_task); + + if (result < 0) { return -1; } - - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(handle, table_ptr, &ref_cnt)) { - return -1; - } - - if (ref_cnt) { - // if 'ref_cnt=0' it's a set dummy marker - - if (parse_task( - handle, - offsets, - async_offsets, - key_addr, - awaited_by, - recurse_task - ) - ) { - return -1; - } - - if (++els == num_els) { - break; - } - } + if (result > 0) { + els++; } table_ptr += sizeof(void*) * 2; @@ -714,81 +960,224 @@ parse_tasks_in_set( static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task +setup_async_result_structure(PyObject **result, PyObject **calls) +{ + *result = PyList_New(1); + if (*result == NULL) { + return -1; + } + + *calls = PyList_New(0); + if (*calls == NULL) { + Py_DECREF(*result); + *result = NULL; + return -1; + } + + if (PyList_SetItem(*result, 0, *calls)) { /* steals ref to 'calls' */ + Py_DECREF(*calls); + Py_DECREF(*result); + *result = NULL; + *calls = NULL; + return -1; + } + + return 0; +} + +static int +add_task_info_to_result( + RemoteUnwinderObject *self, + PyObject *result, + uintptr_t running_task_addr ) { - uintptr_t task_ab_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &task_ab_addr); - if (err) { + PyObject *tn = parse_task_name(self, running_task_addr); + if (tn == NULL) { return -1; } - if ((void*)task_ab_addr == NULL) { - return 0; + if (PyList_Append(result, tn)) { + Py_DECREF(tn); + return -1; } + Py_DECREF(tn); - char awaited_by_is_a_set; - err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, - &awaited_by_is_a_set); - if (err) { + PyObject* awaited_by = PyList_New(0); + if (awaited_by == NULL) { return -1; } - if (awaited_by_is_a_set) { - if (parse_tasks_in_set( - handle, - offsets, - async_offsets, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - awaited_by, - recurse_task - ) - ) { - return -1; - } - } else { - uintptr_t sub_task; - if (read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &sub_task) - ) { - return -1; - } + if (PyList_Append(result, awaited_by)) { + Py_DECREF(awaited_by); + return -1; + } + Py_DECREF(awaited_by); - if (parse_task( - handle, - offsets, - async_offsets, - sub_task, - awaited_by, - recurse_task - ) - ) { - return -1; - } + if (parse_task_awaited_by( + self, running_task_addr, awaited_by, 1) < 0) { + return -1; } return 0; } -typedef struct +static int +process_single_task_node( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject *result +) { + PyObject *tn = NULL; + PyObject *current_awaited_by = NULL; + PyObject *task_id = NULL; + PyObject *result_item = NULL; + + tn = parse_task_name(unwinder, task_addr); + if (tn == NULL) { + goto error; + } + + current_awaited_by = PyList_New(0); + if (current_awaited_by == NULL) { + goto error; + } + + task_id = PyLong_FromUnsignedLongLong(task_addr); + if (task_id == NULL) { + goto error; + } + + result_item = PyTuple_New(3); + if (result_item == NULL) { + goto error; + } + + PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref + PyTuple_SET_ITEM(result_item, 1, tn); // steals ref + PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref + + // References transferred to tuple + task_id = NULL; + tn = NULL; + current_awaited_by = NULL; + + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + return -1; + } + Py_DECREF(result_item); + + // Get back current_awaited_by reference for parse_task_awaited_by + current_awaited_by = PyTuple_GET_ITEM(result_item, 2); + if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by, 0) < 0) { + return -1; + } + + return 0; + +error: + Py_XDECREF(tn); + Py_XDECREF(current_awaited_by); + Py_XDECREF(task_id); + Py_XDECREF(result_item); + return -1; +} + +/* ============================================================================ + * TLBC CACHING FUNCTIONS + * ============================================================================ */ + +#ifdef Py_GIL_DISABLED + +typedef struct { + void *tlbc_array; // Local copy of the TLBC array + Py_ssize_t tlbc_array_size; // Size of the TLBC array + uint32_t generation; // Generation when this was cached +} TLBCCacheEntry; + +static void +tlbc_cache_entry_destroy(void *ptr) { - int lineno; - int end_lineno; - int column; - int end_column; -} LocationInfo; + TLBCCacheEntry *entry = (TLBCCacheEntry *)ptr; + if (entry->tlbc_array) { + PyMem_RawFree(entry->tlbc_array); + } + PyMem_RawFree(entry); +} + +static TLBCCacheEntry * +get_tlbc_cache_entry(RemoteUnwinderObject *self, uintptr_t code_addr, uint32_t current_generation) +{ + void *key = (void *)code_addr; + TLBCCacheEntry *entry = _Py_hashtable_get(self->tlbc_cache, key); + + if (entry && entry->generation != current_generation) { + // Entry is stale, remove it by setting to NULL + _Py_hashtable_set(self->tlbc_cache, key, NULL); + entry = NULL; + } + + return entry; +} + +static int +cache_tlbc_array(RemoteUnwinderObject *self, uintptr_t code_addr, uintptr_t tlbc_array_addr, uint32_t generation) +{ + uintptr_t tlbc_array_ptr; + void *tlbc_array = NULL; + TLBCCacheEntry *entry = NULL; + + // Read the TLBC array pointer + if (read_ptr(&self->handle, tlbc_array_addr, &tlbc_array_ptr) != 0 || tlbc_array_ptr == 0) { + return 0; // No TLBC array + } + + // Read the TLBC array size + Py_ssize_t tlbc_size; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&self->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0 || tlbc_size <= 0) { + return 0; // Invalid size + } + + // Allocate and read the entire TLBC array + size_t array_data_size = tlbc_size * sizeof(void*); + tlbc_array = PyMem_RawMalloc(sizeof(Py_ssize_t) + array_data_size); + if (!tlbc_array) { + return -1; // Memory error + } + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&self->handle, tlbc_array_ptr, sizeof(Py_ssize_t) + array_data_size, tlbc_array) != 0) { + PyMem_RawFree(tlbc_array); + return 0; // Read error + } + + // Create cache entry + entry = PyMem_RawMalloc(sizeof(TLBCCacheEntry)); + if (!entry) { + PyMem_RawFree(tlbc_array); + return -1; // Memory error + } + + entry->tlbc_array = tlbc_array; + entry->tlbc_array_size = tlbc_size; + entry->generation = generation; + + // Store in cache + void *key = (void *)code_addr; + if (_Py_hashtable_set(self->tlbc_cache, key, entry) < 0) { + tlbc_cache_entry_destroy(entry); + return -1; // Cache error + } + + return 1; // Success +} + + + +#endif + +/* ============================================================================ + * LINE TABLE PARSING FUNCTIONS + * ============================================================================ */ static int scan_varint(const uint8_t **ptr) @@ -818,7 +1207,6 @@ scan_signed_varint(const uint8_t **ptr) } } - static bool parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) { @@ -863,7 +1251,9 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L } default: { uint8_t second_byte = *(ptr++); - assert((second_byte & 128) == 0); + if ((second_byte & 128) != 0) { + return false; + } info->column = code << 3 | (second_byte >> 4); info->end_column = info->column + (second_byte & 15); break; @@ -877,240 +1267,387 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L return false; } +/* ============================================================================ + * CODE OBJECT AND FRAME PARSING FUNCTIONS + * ============================================================================ */ + static int -read_remote_pointer(proc_handle_t *handle, uintptr_t address, uintptr_t *out_ptr, const char *error_message) +parse_code_object(RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t instruction_pointer, + uintptr_t *previous_frame, + int32_t tlbc_index) { - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void *), out_ptr); - if (bytes_read < 0) { - return -1; + void *key = (void *)address; + CachedCodeMetadata *meta = NULL; + PyObject *func = NULL; + PyObject *file = NULL; + PyObject *linetable = NULL; + PyObject *lineno = NULL; + PyObject *tuple = NULL; + +#ifdef Py_GIL_DISABLED + // In free threading builds, code object addresses might have the low bit set + // as a flag, so we need to mask it off to get the real address + uintptr_t real_address = address & (~1); +#else + uintptr_t real_address = address; +#endif + + if (unwinder && unwinder->code_object_cache != NULL) { + meta = _Py_hashtable_get(unwinder->code_object_cache, key); } - if ((void *)(*out_ptr) == NULL) { - PyErr_SetString(PyExc_RuntimeError, error_message); - return -1; + if (meta == NULL) { + char code_object[SIZEOF_CODE_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0) + { + goto error; + } + + func = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024); + if (!func) { + goto error; + } + + file = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.filename), 1024); + if (!file) { + goto error; + } + + linetable = read_py_bytes(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), 4096); + if (!linetable) { + goto error; + } + + meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata)); + if (!meta) { + goto error; + } + + meta->func_name = func; + meta->file_name = file; + meta->linetable = linetable; + meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); + meta->addr_code_adaptive = real_address + unwinder->debug_offsets.code_object.co_code_adaptive; + + if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { + cached_code_metadata_destroy(meta); + goto error; + } + + // Ownership transferred to meta + func = NULL; + file = NULL; + linetable = NULL; + } + + uintptr_t ip = instruction_pointer; + ptrdiff_t addrq; + +#ifdef Py_GIL_DISABLED + // Handle thread-local bytecode (TLBC) in free threading builds + if (tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + // No TLBC or no unwinder - use main bytecode directly + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + goto done_tlbc; + } + + // Try to get TLBC data from cache (we'll get generation from the caller) + TLBCCacheEntry *tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); + + if (!tlbc_entry) { + // Cache miss - try to read and cache TLBC array + if (cache_tlbc_array(unwinder, real_address, real_address + unwinder->debug_offsets.code_object.co_tlbc, unwinder->tlbc_generation) > 0) { + tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); + } + } + + if (tlbc_entry && tlbc_index < tlbc_entry->tlbc_array_size) { + // Use cached TLBC data + uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t)); + uintptr_t tlbc_bytecode_addr = entries[tlbc_index]; + + if (tlbc_bytecode_addr != 0) { + // Calculate offset from TLBC bytecode + addrq = (uint16_t *)ip - (uint16_t *)tlbc_bytecode_addr; + goto done_tlbc; + } + } + + // Fall back to main bytecode + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + +done_tlbc: +#else + // Non-free-threaded build, always use the main bytecode + (void)tlbc_index; // Suppress unused parameter warning + (void)unwinder; // Suppress unused parameter warning + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; +#endif + ; // Empty statement to avoid C23 extension warning + LocationInfo info = {0}; + bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), + meta->first_lineno, &info); + if (!ok) { + info.lineno = -1; + } + + lineno = PyLong_FromLong(info.lineno); + if (!lineno) { + goto error; + } + + tuple = PyTuple_New(3); + if (!tuple) { + goto error; } + Py_INCREF(meta->func_name); + Py_INCREF(meta->file_name); + PyTuple_SET_ITEM(tuple, 0, meta->func_name); + PyTuple_SET_ITEM(tuple, 1, meta->file_name); + PyTuple_SET_ITEM(tuple, 2, lineno); + + *result = tuple; return 0; + +error: + Py_XDECREF(func); + Py_XDECREF(file); + Py_XDECREF(linetable); + Py_XDECREF(lineno); + Py_XDECREF(tuple); + return -1; } -static int -read_instruction_ptr(proc_handle_t *handle, struct _Py_DebugOffsets *offsets, - uintptr_t current_frame, uintptr_t *instruction_ptr) +/* ============================================================================ + * STACK CHUNK MANAGEMENT FUNCTIONS + * ============================================================================ */ + +static void +cleanup_stack_chunks(StackChunkList *chunks) { - return read_remote_pointer( - handle, - current_frame + offsets->interpreter_frame.instr_ptr, - instruction_ptr, - "No instruction ptr found" - ); + for (size_t i = 0; i < chunks->count; ++i) { + PyMem_RawFree(chunks->chunks[i].local_copy); + } + PyMem_RawFree(chunks->chunks); } static int -parse_code_object(proc_handle_t *handle, - PyObject **result, - struct _Py_DebugOffsets *offsets, - uintptr_t address, - uintptr_t current_frame, - uintptr_t *previous_frame) -{ - uintptr_t addr_func_name, addr_file_name, addr_linetable, instruction_ptr; +process_single_stack_chunk( + proc_handle_t *handle, + uintptr_t chunk_addr, + StackChunkInfo *chunk_info +) { + // Start with default size assumption + size_t current_size = _PY_DATA_STACK_CHUNK_SIZE; - if (read_remote_pointer(handle, address + offsets->code_object.qualname, &addr_func_name, "No function name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.filename, &addr_file_name, "No file name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.linetable, &addr_linetable, "No linetable found") < 0 || - read_instruction_ptr(handle, offsets, current_frame, &instruction_ptr) < 0) { + char *this_chunk = PyMem_RawMalloc(current_size); + if (!this_chunk) { + PyErr_NoMemory(); return -1; } - int firstlineno; - if (_Py_RemoteDebug_ReadRemoteMemory(handle, - address + offsets->code_object.firstlineno, - sizeof(int), - &firstlineno) < 0) { + if (_Py_RemoteDebug_PagedReadRemoteMemory(handle, chunk_addr, current_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); return -1; } - PyObject *py_linetable = read_py_bytes(handle, offsets, addr_linetable); - if (!py_linetable) { - return -1; + // Check actual size and reread if necessary + size_t actual_size = GET_MEMBER(size_t, this_chunk, offsetof(_PyStackChunk, size)); + if (actual_size != current_size) { + this_chunk = PyMem_RawRealloc(this_chunk, actual_size); + if (!this_chunk) { + PyErr_NoMemory(); + return -1; + } + + if (_Py_RemoteDebug_PagedReadRemoteMemory(handle, chunk_addr, actual_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); + return -1; + } + current_size = actual_size; } - uintptr_t addr_code_adaptive = address + offsets->code_object.co_code_adaptive; - ptrdiff_t addrq = (uint16_t *)instruction_ptr - (uint16_t *)addr_code_adaptive; + chunk_info->remote_addr = chunk_addr; + chunk_info->size = current_size; + chunk_info->local_copy = this_chunk; + return 0; +} - LocationInfo info; - parse_linetable(addrq, PyBytes_AS_STRING(py_linetable), firstlineno, &info); - Py_DECREF(py_linetable); // Done with linetable +static int +copy_stack_chunks(RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + StackChunkList *out_chunks) +{ + uintptr_t chunk_addr; + StackChunkInfo *chunks = NULL; + size_t count = 0; + size_t max_chunks = 16; - PyObject *py_line = PyLong_FromLong(info.lineno); - if (!py_line) { + if (read_ptr(&unwinder->handle, tstate_addr + unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { return -1; } - PyObject *py_func_name = read_py_str(handle, offsets, addr_func_name, 256); - if (!py_func_name) { - Py_DECREF(py_line); + chunks = PyMem_RawMalloc(max_chunks * sizeof(StackChunkInfo)); + if (!chunks) { + PyErr_NoMemory(); return -1; } - PyObject *py_file_name = read_py_str(handle, offsets, addr_file_name, 256); - if (!py_file_name) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - return -1; + while (chunk_addr != 0) { + // Grow array if needed + if (count >= max_chunks) { + max_chunks *= 2; + StackChunkInfo *new_chunks = PyMem_RawRealloc(chunks, max_chunks * sizeof(StackChunkInfo)); + if (!new_chunks) { + PyErr_NoMemory(); + goto error; + } + chunks = new_chunks; + } + + // Process this chunk + if (process_single_stack_chunk(&unwinder->handle, chunk_addr, &chunks[count]) < 0) { + goto error; + } + + // Get next chunk address and increment count + chunk_addr = GET_MEMBER(uintptr_t, chunks[count].local_copy, offsetof(_PyStackChunk, previous)); + count++; } - PyObject *result_tuple = PyTuple_New(3); - if (!result_tuple) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - Py_DECREF(py_file_name); - return -1; + out_chunks->chunks = chunks; + out_chunks->count = count; + return 0; + +error: + for (size_t i = 0; i < count; ++i) { + PyMem_RawFree(chunks[i].local_copy); } + PyMem_RawFree(chunks); + return -1; +} - PyTuple_SET_ITEM(result_tuple, 0, py_func_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 1, py_file_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 2, py_line); // steals ref +static void * +find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr) +{ + for (size_t i = 0; i < chunks->count; ++i) { + uintptr_t base = chunks->chunks[i].remote_addr + offsetof(_PyStackChunk, data); + size_t payload = chunks->chunks[i].size - offsetof(_PyStackChunk, data); - *result = result_tuple; - return 0; + if (remote_ptr >= base && remote_ptr < base + payload) { + return (char *)chunks->chunks[i].local_copy + (remote_ptr - chunks->chunks[i].remote_addr); + } + } + return NULL; } static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, +parse_frame_from_chunks( + RemoteUnwinderObject *unwinder, + PyObject **result, uintptr_t address, - uintptr_t* previous_frame + uintptr_t *previous_frame, + StackChunkList *chunks ) { - int err; - - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { + void *frame_ptr = find_frame_in_chunks(chunks, address); + if (!frame_ptr) { return -1; } - char owner; - if (read_char(handle, address + offsets->interpreter_frame.owner, &owner)) { - return -1; - } + char *frame = (char *)frame_ptr; + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); - if (owner >= FRAME_OWNED_BY_INTERPRETER) { + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) >= FRAME_OWNED_BY_INTERPRETER || + !GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable)) { return 0; } - uintptr_t address_of_code_object; - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - &address_of_code_object - ); - if (err) { - return -1; - } + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); - if ((void*)address_of_code_object == NULL) { - return 0; + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); } +#endif return parse_code_object( - handle, result, offsets, address_of_code_object, address, previous_frame); + unwinder, result, GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable), + instruction_pointer, previous_frame, tlbc_index); } +/* ============================================================================ + * INTERPRETER STATE AND THREAD DISCOVERY FUNCTIONS + * ============================================================================ */ + static int -parse_async_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame, - uintptr_t* code_object +populate_initial_state_data( + int all_threads, + RemoteUnwinderObject *unwinder, + uintptr_t runtime_start_address, + uintptr_t *interpreter_state, + uintptr_t *tstate ) { - int err; - - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { - return -1; - } + uint64_t interpreter_state_list_head = + unwinder->debug_offsets.runtime_state.interpreters_head; - char owner; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->interpreter_frame.owner, sizeof(char), &owner); + uintptr_t address_of_interpreter_state; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + runtime_start_address + interpreter_state_list_head, + sizeof(void*), + &address_of_interpreter_state); if (bytes_read < 0) { return -1; } - if (owner == FRAME_OWNED_BY_CSTACK || owner == FRAME_OWNED_BY_INTERPRETER) { - return 0; // C frame - } - - if (owner != FRAME_OWNED_BY_GENERATOR - && owner != FRAME_OWNED_BY_THREAD) { - PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); + if (address_of_interpreter_state == 0) { + PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); return -1; } - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - code_object - ); - if (err) { - return -1; - } + *interpreter_state = address_of_interpreter_state; - assert(code_object != NULL); - if ((void*)*code_object == NULL) { + if (all_threads) { + *tstate = 0; return 0; } - if (parse_code_object( - handle, result, offsets, *code_object, address, previous_frame)) { - return -1; - } - - return 1; -} + uintptr_t address_of_thread = address_of_interpreter_state + + unwinder->debug_offsets.interpreter_state.threads_main; -static int -read_async_debug( - proc_handle_t *handle, - struct _Py_AsyncioModuleDebugOffsets* async_debug -) { - uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(handle); - if (!async_debug_addr) { + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address_of_thread, + sizeof(void*), + tstate) < 0) { return -1; } - size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, async_debug_addr, size, async_debug); - return result; + return 0; } static int find_running_frame( - proc_handle_t *handle, + RemoteUnwinderObject *unwinder, uintptr_t runtime_start_address, - _Py_DebugOffsets* local_debug_offsets, uintptr_t *frame ) { uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; + unwinder->debug_offsets.runtime_state.interpreters_head; uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); @@ -1124,10 +1661,10 @@ find_running_frame( } uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, + bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_main, + unwinder->debug_offsets.interpreter_state.threads_main, sizeof(void*), &address_of_thread); if (bytes_read < 0) { @@ -1137,8 +1674,8 @@ find_running_frame( // No Python frames are available for us (can happen at tear-down). if ((void*)address_of_thread != NULL) { int err = read_ptr( - handle, - address_of_thread + local_debug_offsets->thread_state.current_frame, + &unwinder->handle, + address_of_thread + unwinder->debug_offsets.thread_state.current_frame, frame); if (err) { return -1; @@ -1152,21 +1689,18 @@ find_running_frame( static int find_running_task( - proc_handle_t *handle, - uintptr_t runtime_start_address, - _Py_DebugOffsets *local_debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, + RemoteUnwinderObject *unwinder, uintptr_t *running_task_addr ) { *running_task_addr = (uintptr_t)NULL; uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; + unwinder->debug_offsets.runtime_state.interpreters_head; uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_address + interpreter_state_list_head, + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + unwinder->runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); if (bytes_read < 0) { @@ -1179,10 +1713,10 @@ find_running_task( } uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, + bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_head, + unwinder->debug_offsets.interpreter_state.threads_head, sizeof(void*), &address_of_thread); if (bytes_read < 0) { @@ -1196,9 +1730,9 @@ find_running_task( } bytes_read = read_py_ptr( - handle, + &unwinder->handle, address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_loop, + + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, &address_of_running_loop); if (bytes_read == -1) { return -1; @@ -1210,9 +1744,9 @@ find_running_task( } int err = read_ptr( - handle, + &unwinder->handle, address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_task, + + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, running_task_addr); if (err) { return -1; @@ -1222,579 +1756,936 @@ find_running_task( } static int -append_awaited_by_for_thread( - proc_handle_t *handle, - uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - PyObject *result +find_running_task_and_coro( + RemoteUnwinderObject *self, + uintptr_t *running_task_addr, + uintptr_t *running_coro_addr, + uintptr_t *running_task_code_obj ) { - struct llist_node task_node; - - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - head_addr, - sizeof(task_node), - &task_node)) - { + *running_task_addr = (uintptr_t)NULL; + if (find_running_task( + self, running_task_addr) < 0) { + chain_exceptions(PyExc_RuntimeError, "Failed to find running task"); return -1; } - size_t iteration_count = 0; - const size_t MAX_ITERATIONS = 2 << 15; // A reasonable upper bound - while ((uintptr_t)task_node.next != head_addr) { - if (++iteration_count > MAX_ITERATIONS) { - PyErr_SetString(PyExc_RuntimeError, "Task list appears corrupted"); - return -1; - } - - if (task_node.next == NULL) { - PyErr_SetString( - PyExc_RuntimeError, - "Invalid linked list structure reading remote memory"); - return -1; - } - - uintptr_t task_addr = (uintptr_t)task_node.next - - async_offsets->asyncio_task_object.task_node; - - PyObject *tn = parse_task_name( - handle, - debug_offsets, - async_offsets, - task_addr); - if (tn == NULL) { - return -1; - } - - PyObject *current_awaited_by = PyList_New(0); - if (current_awaited_by == NULL) { - Py_DECREF(tn); - return -1; - } - - PyObject* task_id = PyLong_FromUnsignedLongLong(task_addr); - if (task_id == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); - return -1; - } - - PyObject *result_item = PyTuple_New(3); - if (result_item == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); - Py_DECREF(task_id); - return -1; - } - - PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref - PyTuple_SET_ITEM(result_item, 1, tn); // steals ref - PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - return -1; - } - Py_DECREF(result_item); - - if (parse_task_awaited_by(handle, debug_offsets, async_offsets, - task_addr, current_awaited_by, 0)) - { - return -1; - } - - // onto the next one... - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - (uintptr_t)task_node.next, - sizeof(task_node), - &task_node)) - { - return -1; - } - } - - return 0; -} - -static int -append_awaited_by( - proc_handle_t *handle, - unsigned long tid, - uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - PyObject *result) -{ - PyObject *tid_py = PyLong_FromUnsignedLong(tid); - if (tid_py == NULL) { + if ((void*)*running_task_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "No running task found"); return -1; } - PyObject *result_item = PyTuple_New(2); - if (result_item == NULL) { - Py_DECREF(tid_py); + if (read_py_ptr( + &self->handle, + *running_task_addr + self->async_debug_offsets.asyncio_task_object.task_coro, + running_coro_addr) < 0) { + chain_exceptions(PyExc_RuntimeError, "Failed to read running task coro"); return -1; } - PyObject* awaited_by_for_thread = PyList_New(0); - if (awaited_by_for_thread == NULL) { - Py_DECREF(tid_py); - Py_DECREF(result_item); + if ((void*)*running_coro_addr == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); return -1; } - PyTuple_SET_ITEM(result_item, 0, tid_py); // steals ref - PyTuple_SET_ITEM(result_item, 1, awaited_by_for_thread); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + if (read_py_ptr( + &self->handle, + *running_coro_addr + self->debug_offsets.gen_object.gi_iframe, + running_task_code_obj) < 0) { return -1; } - Py_DECREF(result_item); - if (append_awaited_by_for_thread( - handle, - head_addr, - debug_offsets, - async_offsets, - awaited_by_for_thread)) - { + if ((void*)*running_task_code_obj == NULL) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); return -1; } return 0; } -static PyObject* -get_all_awaited_by(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_all_awaited_by is not implemented on this platform"); - return NULL; -#endif - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; +/* ============================================================================ + * FRAME PARSING FUNCTIONS + * ============================================================================ */ + +static int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + + Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + return -1; } - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) >= FRAME_OWNED_BY_INTERPRETER) { return 0; } - PyObject *result = NULL; - - uintptr_t runtime_start_addr = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_addr == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; + if ((void*)GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable) == NULL) { + return 0; } - struct _Py_DebugOffsets local_debug_offsets; - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_addr, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); + + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); } +#endif - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; + return parse_code_object( + unwinder, result, GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable), + instruction_pointer, previous_frame, tlbc_index); +} + +static int +parse_async_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* previous_frame, + uintptr_t* code_object +) { + char frame[SIZEOF_INTERP_FRAME]; + + Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); + if (bytes_read < 0) { + return -1; } - result = PyList_New(0); - if (result == NULL) { - goto result_err; + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_CSTACK || + GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { + return 0; // C frame } - uint64_t interpreter_state_list_head = - local_debug_offsets.runtime_state.interpreters_head; + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR + && GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", + GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner)); + return -1; + } - uintptr_t interpreter_state_addr; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_addr + interpreter_state_list_head, - sizeof(void*), - &interpreter_state_addr)) - { - goto result_err; + *code_object = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); + // Strip tag bits for consistent comparison + *code_object &= ~Py_TAG_BITS; + + assert(code_object != NULL); + if ((void*)*code_object == NULL) { + return 0; } - uintptr_t thread_state_addr; - unsigned long tid = 0; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - interpreter_state_addr - + local_debug_offsets.interpreter_state.threads_head, - sizeof(void*), - &thread_state_addr)) - { - goto result_err; + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); + + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); } +#endif - uintptr_t head_addr; - while (thread_state_addr != 0) { - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr - + local_debug_offsets.thread_state.native_thread_id, - sizeof(tid), - &tid)) - { - goto result_err; + if (parse_code_object( + unwinder, result, *code_object, instruction_pointer, previous_frame, tlbc_index)) { + return -1; + } + + return 1; +} + +static int +parse_async_frame_chain( + RemoteUnwinderObject *self, + PyObject *calls, + uintptr_t running_task_code_obj +) { + uintptr_t address_of_current_frame; + if (find_running_frame(self, self->runtime_start_address, &address_of_current_frame) < 0) { + chain_exceptions(PyExc_RuntimeError, "Failed to find running frame"); + return -1; + } + + uintptr_t address_of_code_object; + while ((void*)address_of_current_frame != NULL) { + PyObject* frame_info = NULL; + int res = parse_async_frame_object( + self, + &frame_info, + address_of_current_frame, + &address_of_current_frame, + &address_of_code_object + ); + + if (res < 0) { + chain_exceptions(PyExc_RuntimeError, "Failed to parse async frame object"); + return -1; } - head_addr = thread_state_addr - + local_async_debug.asyncio_thread_state.asyncio_tasks_head; + if (!frame_info) { + continue; + } - if (append_awaited_by(handle, tid, head_addr, &local_debug_offsets, - &local_async_debug, result)) - { - goto result_err; + if (PyList_Append(calls, frame_info) == -1) { + Py_DECREF(frame_info); + return -1; } - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr + local_debug_offsets.thread_state.next, - sizeof(void*), - &thread_state_addr)) - { - goto result_err; + Py_DECREF(frame_info); + + if (address_of_code_object == running_task_code_obj) { + break; } } - head_addr = interpreter_state_addr - + local_async_debug.asyncio_interpreter_state.asyncio_tasks_head; + return 0; +} - // On top of a per-thread task lists used by default by asyncio to avoid - // contention, there is also a fallback per-interpreter list of tasks; - // any tasks still pending when a thread is destroyed will be moved to the - // per-interpreter task list. It's unlikely we'll find anything here, but - // interesting for debugging. - if (append_awaited_by(handle, 0, head_addr, &local_debug_offsets, - &local_async_debug, result)) - { - goto result_err; +/* ============================================================================ + * AWAITED BY PARSING FUNCTIONS + * ============================================================================ */ + +static int +append_awaited_by_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t head_addr, + PyObject *result +) { + char task_node[SIZEOF_LLIST_NODE]; + + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, head_addr, + sizeof(task_node), task_node) < 0) { + return -1; } - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; + size_t iteration_count = 0; + const size_t MAX_ITERATIONS = 2 << 15; // A reasonable upper bound -result_err: - Py_XDECREF(result); - _Py_RemoteDebug_CleanupProcHandle(handle); - return NULL; -} + while (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) != head_addr) { + if (++iteration_count > MAX_ITERATIONS) { + PyErr_SetString(PyExc_RuntimeError, "Task list appears corrupted"); + return -1; + } -static PyObject* -get_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif + if (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) == 0) { + PyErr_SetString(PyExc_RuntimeError, + "Invalid linked list structure reading remote memory"); + return -1; + } - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; - } + uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) + - unwinder->async_debug_offsets.asyncio_task_object.task_node; - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; + if (process_single_task_node(unwinder, task_addr, result) < 0) { + return -1; + } + + // Read next node + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next), + sizeof(task_node), + task_node) < 0) { + return -1; + } } - PyObject* result = NULL; + return 0; +} - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; +static int +append_awaited_by( + RemoteUnwinderObject *unwinder, + unsigned long tid, + uintptr_t head_addr, + PyObject *result) +{ + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + return -1; } - struct _Py_DebugOffsets local_debug_offsets; - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; + PyObject *result_item = PyTuple_New(2); + if (result_item == NULL) { + Py_DECREF(tid_py); + return -1; } - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - goto result_err; + PyObject* awaited_by_for_thread = PyList_New(0); + if (awaited_by_for_thread == NULL) { + Py_DECREF(tid_py); + Py_DECREF(result_item); + return -1; } - result = PyList_New(0); - if (result == NULL) { - goto result_err; + PyTuple_SET_ITEM(result_item, 0, tid_py); // steals ref + PyTuple_SET_ITEM(result_item, 1, awaited_by_for_thread); // steals ref + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + return -1; } + Py_DECREF(result_item); - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - if (parse_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_CLEAR(result); - goto result_err; + if (append_awaited_by_for_thread(unwinder, head_addr, awaited_by_for_thread)) + { + return -1; + } + + return 0; +} + +/* ============================================================================ + * STACK UNWINDING FUNCTIONS + * ============================================================================ */ + +static int +process_frame_chain( + RemoteUnwinderObject *unwinder, + uintptr_t initial_frame_addr, + StackChunkList *chunks, + PyObject *frame_info +) { + uintptr_t frame_addr = initial_frame_addr; + uintptr_t prev_frame_addr = 0; + const size_t MAX_FRAMES = 1024; + size_t frame_count = 0; + + while ((void*)frame_addr != NULL) { + PyObject *frame = NULL; + uintptr_t next_frame_addr = 0; + + if (++frame_count > MAX_FRAMES) { + PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); + return -1; } - if (!frame_info) { - continue; + // Try chunks first, fallback to direct memory read + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { + PyErr_Clear(); + if (parse_frame_object(unwinder, &frame, frame_addr, &next_frame_addr) < 0) { + return -1; + } } - if (PyList_Append(result, frame_info) == -1) { - Py_CLEAR(result); - goto result_err; + if (!frame) { + break; } - Py_DECREF(frame_info); - frame_info = NULL; + if (prev_frame_addr && frame_addr != prev_frame_addr) { + PyErr_Format(PyExc_RuntimeError, + "Broken frame chain: expected frame at 0x%lx, got 0x%lx", + prev_frame_addr, frame_addr); + Py_DECREF(frame); + return -1; + } + if (PyList_Append(frame_info, frame) == -1) { + Py_DECREF(frame); + return -1; + } + Py_DECREF(frame); + + prev_frame_addr = next_frame_addr; + frame_addr = next_frame_addr; } -result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; + return 0; } static PyObject* -get_async_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif - int pid; +unwind_stack_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t *current_tstate +) { + PyObject *frame_info = NULL; + PyObject *thread_id = NULL; + PyObject *result = NULL; + StackChunkList chunks = {0}; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; + char ts[SIZEOF_THREAD_STATE]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, *current_tstate, unwinder->debug_offsets.thread_state.size, ts); + if (bytes_read < 0) { + goto error; } - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; - } + uintptr_t frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.current_frame); - PyObject *result = NULL; + frame_info = PyList_New(0); + if (!frame_info) { + goto error; + } - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; + if (copy_stack_chunks(unwinder, *current_tstate, &chunks) < 0) { + goto error; } - struct _Py_DebugOffsets local_debug_offsets; - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info) < 0) { + goto error; } - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; + *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); + + thread_id = PyLong_FromLongLong( + GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id)); + if (thread_id == NULL) { + goto error; } - result = PyList_New(1); + result = PyTuple_New(2); if (result == NULL) { - goto result_err; + goto error; } - PyObject* calls = PyList_New(0); - if (calls == NULL) { - goto result_err; + + PyTuple_SET_ITEM(result, 0, thread_id); // Steals reference + PyTuple_SET_ITEM(result, 1, frame_info); // Steals reference + + cleanup_stack_chunks(&chunks); + return result; + +error: + Py_XDECREF(frame_info); + Py_XDECREF(thread_id); + Py_XDECREF(result); + cleanup_stack_chunks(&chunks); + return NULL; +} + + +/* ============================================================================ + * REMOTEUNWINDER CLASS IMPLEMENTATION + * ============================================================================ */ + +/*[clinic input] +class _remote_debugging.RemoteUnwinder "RemoteUnwinderObject *" "&RemoteUnwinder_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=55f164d8803318be]*/ + +/*[clinic input] +_remote_debugging.RemoteUnwinder.__init__ + pid: int + * + all_threads: bool = False + +Initialize a new RemoteUnwinder object for debugging a remote Python process. + +Args: + pid: Process ID of the target Python process to debug + all_threads: If True, initialize state for all threads in the process. + If False, only initialize for the main thread. + +The RemoteUnwinder provides functionality to inspect and debug a running Python +process, including examining thread states, stack frames and other runtime data. + +Raises: + PermissionError: If access to the target process is denied + OSError: If unable to attach to the target process or access its memory + RuntimeError: If unable to read debug information from the target process +[clinic start generated code]*/ + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads) +/*[clinic end generated code: output=b8027cb247092081 input=6a2056b04e6f050e]*/ +{ + if (_Py_RemoteDebug_InitProcHandle(&self->handle, pid) < 0) { + return -1; } - if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ - Py_DECREF(calls); - goto result_err; + + self->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&self->handle); + if (self->runtime_start_address == 0) { + return -1; } - uintptr_t running_task_addr = (uintptr_t)NULL; - if (find_running_task( - handle, runtime_start_address, &local_debug_offsets, &local_async_debug, - &running_task_addr) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running task"); - goto result_err; + if (_Py_RemoteDebug_ReadDebugOffsets(&self->handle, + &self->runtime_start_address, + &self->debug_offsets) < 0) + { + return -1; } - if ((void*)running_task_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No running task found"); - goto result_err; + // Try to read async debug offsets, but don't fail if they're not available + self->async_debug_offsets_available = 1; + if (read_async_debug(self) < 0) { + PyErr_Clear(); + memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); + self->async_debug_offsets_available = 0; } - uintptr_t running_coro_addr; - if (read_py_ptr( - handle, - running_task_addr + local_async_debug.asyncio_task_object.task_coro, - &running_coro_addr - )) { - chain_exceptions(PyExc_RuntimeError, "Failed to read running task coro"); - goto result_err; + if (populate_initial_state_data(all_threads, self, self->runtime_start_address, + &self->interpreter_addr ,&self->tstate_addr) < 0) + { + return -1; } - if ((void*)running_coro_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); - goto result_err; + self->code_object_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + cached_code_metadata_destroy, + NULL + ); + if (self->code_object_cache == NULL) { + PyErr_NoMemory(); + return -1; } - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - uintptr_t address_of_running_task_code_obj; - if (read_py_ptr( - handle, - running_coro_addr + local_debug_offsets.gen_object.gi_iframe, - &address_of_running_task_code_obj - )) { - goto result_err; +#ifdef Py_GIL_DISABLED + // Initialize TLBC cache + self->tlbc_generation = 0; + self->tlbc_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + tlbc_cache_entry_destroy, + NULL + ); + if (self->tlbc_cache == NULL) { + _Py_hashtable_destroy(self->code_object_cache); + PyErr_NoMemory(); + return -1; } +#endif - if ((void*)address_of_running_task_code_obj == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - goto result_err; + return 0; +} + +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_stack_trace + +Returns a list of stack traces for all threads in the target process. + +Each element in the returned list is a tuple of (thread_id, frame_list), where: +- thread_id is the OS thread identifier +- frame_list is a list of tuples (function_name, filename, line_number) representing + the Python stack frames for that thread, ordered from most recent to oldest + +Example: + [ + (1234, [ + ('process_data', 'worker.py', 127), + ('run_worker', 'worker.py', 45), + ('main', 'app.py', 23) + ]), + (1235, [ + ('handle_request', 'server.py', 89), + ('serve_forever', 'server.py', 52) + ]) + ] + +Raises: + RuntimeError: If there is an error copying memory from the target process + OSError: If there is an error accessing the target process + PermissionError: If access to the target process is denied + UnicodeDecodeError: If there is an error decoding strings from the target process + +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=666192b90c69d567 input=331dbe370578badf]*/ +{ + PyObject* result = NULL; + // Read interpreter state into opaque buffer + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + self->interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + goto exit; } - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running frame"); - goto result_err; + // Get code object generation from buffer + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (code_object_generation != self->code_object_generation) { + self->code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); } - uintptr_t address_of_code_object; - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - int res = parse_async_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame, - &address_of_code_object - ); +#ifdef Py_GIL_DISABLED + // Check TLBC generation and invalidate cache if needed + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } +#endif - if (res < 0) { - chain_exceptions(PyExc_RuntimeError, "Failed to parse async frame object"); - goto result_err; - } + uintptr_t current_tstate; + if (self->tstate_addr == 0) { + // Get threads head from buffer + current_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.threads_head); + } else { + current_tstate = self->tstate_addr; + } + result = PyList_New(0); + if (!result) { + goto exit; + } + + while (current_tstate != 0) { + PyObject* frame_info = unwind_stack_for_thread(self, ¤t_tstate); if (!frame_info) { - continue; + Py_CLEAR(result); + goto exit; } - if (PyList_Append(calls, frame_info) == -1) { - Py_DECREF(calls); - goto result_err; + if (PyList_Append(result, frame_info) == -1) { + Py_DECREF(frame_info); + Py_CLEAR(result); + goto exit; } - Py_DECREF(frame_info); - frame_info = NULL; - if (address_of_code_object == address_of_running_task_code_obj) { + // We are targeting a single tstate, break here + if (self->tstate_addr) { break; } } - PyObject *tn = parse_task_name( - handle, &local_debug_offsets, &local_async_debug, running_task_addr); - if (tn == NULL) { - goto result_err; +exit: + _Py_RemoteDebug_ClearCache(&self->handle); + return result; +} + +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_all_awaited_by + +Get all tasks and their awaited_by relationships from the remote process. + +This provides a tree structure showing which tasks are waiting for other tasks. + +For each task, returns: +1. The call stack frames leading to where the task is currently executing +2. The name of the task +3. A list of tasks that this task is waiting for, with their own frames/names/etc + +Returns a list of [frames, task_name, subtasks] where: +- frames: List of (func_name, filename, lineno) showing the call stack +- task_name: String identifier for the task +- subtasks: List of tasks being awaited by this task, in same format + +Raises: + RuntimeError: If AsyncioDebug section is not available in the remote process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output: +[ + # Task c2_root waiting for two subtasks + [ + # Call stack of c2_root + [("c5", "script.py", 10), ("c4", "script.py", 14)], + "c2_root", + [ + # First subtask (sub_main_2) and what it's waiting for + [ + [("c1", "script.py", 23)], + "sub_main_2", + [...] + ], + # Second subtask and its waiters + [...] + ] + ] +] +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6a49cd345e8aec53 input=a452c652bb00701a]*/ +{ + if (!self->async_debug_offsets_available) { + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + return NULL; } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); + + PyObject *result = PyList_New(0); + if (result == NULL) { goto result_err; } - Py_DECREF(tn); - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { + uintptr_t thread_state_addr; + unsigned long tid = 0; + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + self->interpreter_addr + + self->debug_offsets.interpreter_state.threads_main, + sizeof(void*), + &thread_state_addr)) + { goto result_err; } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto result_err; + + uintptr_t head_addr; + while (thread_state_addr != 0) { + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + thread_state_addr + + self->debug_offsets.thread_state.native_thread_id, + sizeof(tid), + &tid)) + { + goto result_err; + } + + head_addr = thread_state_addr + + self->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + + if (append_awaited_by(self, tid, head_addr, result)) + { + goto result_err; + } + + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + thread_state_addr + self->debug_offsets.thread_state.next, + sizeof(void*), + &thread_state_addr)) + { + goto result_err; + } } - Py_DECREF(awaited_by); - if (parse_task_awaited_by( - handle, &local_debug_offsets, &local_async_debug, - running_task_addr, awaited_by, 1) - ) { + head_addr = self->interpreter_addr + + self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; + + // On top of a per-thread task lists used by default by asyncio to avoid + // contention, there is also a fallback per-interpreter list of tasks; + // any tasks still pending when a thread is destroyed will be moved to the + // per-interpreter task list. It's unlikely we'll find anything here, but + // interesting for debugging. + if (append_awaited_by(self, 0, head_addr, result)) + { goto result_err; } - _Py_RemoteDebug_CleanupProcHandle(handle); + _Py_RemoteDebug_ClearCache(&self->handle); return result; result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_async_stack_trace -static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, - "Get the Python stack from a given pod"}, - {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, - "Get the asyncio stack from a given pid"}, - {"get_all_awaited_by", get_all_awaited_by, METH_VARARGS, - "Get all tasks and their awaited_by from a given pid"}, - {NULL, NULL, 0, NULL}, +Returns information about the currently running async task and its stack trace. + +Returns a tuple of (task_info, stack_frames) where: +- task_info is a tuple of (task_id, task_name) identifying the task +- stack_frames is a list of tuples (function_name, filename, line_number) representing + the Python stack frames for the task, ordered from most recent to oldest + +Example: + ((4345585712, 'Task-1'), [ + ('run_echo_server', 'server.py', 127), + ('serve_forever', 'server.py', 45), + ('main', 'app.py', 23) + ]) + +Raises: + RuntimeError: If AsyncioDebug section is not available in the target process + RuntimeError: If there is an error copying memory from the target process + OSError: If there is an error accessing the target process + PermissionError: If access to the target process is denied + UnicodeDecodeError: If there is an error decoding strings from the target process + +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6433d52b55e87bbe input=11b7150c59d4c60f]*/ +{ + if (!self->async_debug_offsets_available) { + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + return NULL; + } + + PyObject *result = NULL; + PyObject *calls = NULL; + + if (setup_async_result_structure(&result, &calls) < 0) { + goto cleanup; + } + + uintptr_t running_task_addr, running_coro_addr, running_task_code_obj; + if (find_running_task_and_coro(self, &running_task_addr, + &running_coro_addr, &running_task_code_obj) < 0) { + goto cleanup; + } + + if (parse_async_frame_chain(self, calls, running_task_code_obj) < 0) { + goto cleanup; + } + + if (add_task_info_to_result(self, result, running_task_addr) < 0) { + goto cleanup; + } + + _Py_RemoteDebug_ClearCache(&self->handle); + return result; + +cleanup: + _Py_RemoteDebug_ClearCache(&self->handle); + Py_XDECREF(result); + return NULL; +} + +static PyMethodDef RemoteUnwinder_methods[] = { + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF + {NULL, NULL} }; -static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, - .m_name = "_remote_debugging", - .m_size = -1, - .m_methods = methods, +static void +RemoteUnwinder_dealloc(RemoteUnwinderObject *self) +{ + PyTypeObject *tp = Py_TYPE(self); + if (self->code_object_cache) { + _Py_hashtable_destroy(self->code_object_cache); + } +#ifdef Py_GIL_DISABLED + if (self->tlbc_cache) { + _Py_hashtable_destroy(self->tlbc_cache); + } +#endif + if (self->handle.pid != 0) { + _Py_RemoteDebug_ClearCache(&self->handle); + _Py_RemoteDebug_CleanupProcHandle(&self->handle); + } + PyObject_Del(self); + Py_DECREF(tp); +} + +static PyType_Slot RemoteUnwinder_slots[] = { + {Py_tp_doc, (void *)"RemoteUnwinder(pid): Inspect stack of a remote Python process."}, + {Py_tp_methods, RemoteUnwinder_methods}, + {Py_tp_init, _remote_debugging_RemoteUnwinder___init__}, + {Py_tp_dealloc, RemoteUnwinder_dealloc}, + {0, NULL} }; -PyMODINIT_FUNC -PyInit__remote_debugging(void) +static PyType_Spec RemoteUnwinder_spec = { + .name = "_remote_debugging.RemoteUnwinder", + .basicsize = sizeof(RemoteUnwinderObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = RemoteUnwinder_slots, +}; + +/* ============================================================================ + * MODULE INITIALIZATION + * ============================================================================ */ + +static int +_remote_debugging_exec(PyObject *m) { - PyObject* mod = PyModule_Create(&module); - if (mod == NULL) { - return NULL; + RemoteDebuggingState *st = RemoteDebugging_GetState(m); +#define CREATE_TYPE(mod, type, spec) \ + do { \ + type = (PyTypeObject *)PyType_FromMetaclass(NULL, mod, spec, NULL); \ + if (type == NULL) { \ + return -1; \ + } \ + } while (0) + + CREATE_TYPE(m, st->RemoteDebugging_Type, &RemoteUnwinder_spec); + + if (PyModule_AddType(m, st->RemoteDebugging_Type) < 0) { + return -1; } #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); #endif - int rc = PyModule_AddIntConstant( - mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + int rc = PyModule_AddIntConstant(m, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); if (rc < 0) { - Py_DECREF(mod); - return NULL; + return -1; } - return mod; + if (RemoteDebugging_InitState(st) < 0) { + return -1; + } + return 0; } + +static int +remote_debugging_traverse(PyObject *mod, visitproc visit, void *arg) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_VISIT(state->RemoteDebugging_Type); + return 0; +} + +static int +remote_debugging_clear(PyObject *mod) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_CLEAR(state->RemoteDebugging_Type); + return 0; +} + +static void +remote_debugging_free(void *mod) +{ + (void)remote_debugging_clear((PyObject *)mod); +} + +static PyModuleDef_Slot remote_debugging_slots[] = { + {Py_mod_exec, _remote_debugging_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; + +static PyMethodDef remote_debugging_methods[] = { + {NULL, NULL, 0, NULL}, +}; + +static struct PyModuleDef remote_debugging_module = { + PyModuleDef_HEAD_INIT, + .m_name = "_remote_debugging", + .m_size = sizeof(RemoteDebuggingState), + .m_methods = remote_debugging_methods, + .m_slots = remote_debugging_slots, + .m_traverse = remote_debugging_traverse, + .m_clear = remote_debugging_clear, + .m_free = remote_debugging_free, +}; + +PyMODINIT_FUNC +PyInit__remote_debugging(void) +{ + return PyModuleDef_Init(&remote_debugging_module); +} + diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h new file mode 100644 index 00000000000000..e83e2fd7fd2b5b --- /dev/null +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -0,0 +1,243 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, +"RemoteUnwinder(pid, *, all_threads=False)\n" +"--\n" +"\n" +"Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" +"\n" +"Args:\n" +" pid: Process ID of the target Python process to debug\n" +" all_threads: If True, initialize state for all threads in the process.\n" +" If False, only initialize for the main thread.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" +"process, including examining thread states, stack frames and other runtime data.\n" +"\n" +"Raises:\n" +" PermissionError: If access to the target process is denied\n" +" OSError: If unable to attach to the target process or access its memory\n" +" RuntimeError: If unable to read debug information from the target process"); + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads); + +static int +_remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pid", "all_threads", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "RemoteUnwinder", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + int pid; + int all_threads = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + pid = PyLong_AsInt(fastargs[0]); + if (pid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + all_threads = PyObject_IsTrue(fastargs[1]); + if (all_threads < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads); + +exit: + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, +"get_stack_trace($self, /)\n" +"--\n" +"\n" +"Returns a list of stack traces for all threads in the target process.\n" +"\n" +"Each element in the returned list is a tuple of (thread_id, frame_list), where:\n" +"- thread_id is the OS thread identifier\n" +"- frame_list is a list of tuples (function_name, filename, line_number) representing\n" +" the Python stack frames for that thread, ordered from most recent to oldest\n" +"\n" +"Example:\n" +" [\n" +" (1234, [\n" +" (\'process_data\', \'worker.py\', 127),\n" +" (\'run_worker\', \'worker.py\', 45),\n" +" (\'main\', \'app.py\', 23)\n" +" ]),\n" +" (1235, [\n" +" (\'handle_request\', \'server.py\', 89),\n" +" (\'serve_forever\', \'server.py\', 52)\n" +" ])\n" +" ]\n" +"\n" +"Raises:\n" +" RuntimeError: If there is an error copying memory from the target process\n" +" OSError: If there is an error accessing the target process\n" +" PermissionError: If access to the target process is denied\n" +" UnicodeDecodeError: If there is an error decoding strings from the target process"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ + {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, +"get_all_awaited_by($self, /)\n" +"--\n" +"\n" +"Get all tasks and their awaited_by relationships from the remote process.\n" +"\n" +"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"\n" +"For each task, returns:\n" +"1. The call stack frames leading to where the task is currently executing\n" +"2. The name of the task\n" +"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"\n" +"Returns a list of [frames, task_name, subtasks] where:\n" +"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- task_name: String identifier for the task\n" +"- subtasks: List of tasks being awaited by this task, in same format\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output:\n" +"[\n" +" [\n" +" [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n" +" \"c2_root\",\n" +" [\n" +" [\n" +" [(\"c1\", \"script.py\", 23)],\n" +" \"sub_main_2\",\n" +" [...]\n" +" ],\n" +" [...]\n" +" ]\n" +" ]\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF \ + {"get_all_awaited_by", (PyCFunction)_remote_debugging_RemoteUnwinder_get_all_awaited_by, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, +"get_async_stack_trace($self, /)\n" +"--\n" +"\n" +"Returns information about the currently running async task and its stack trace.\n" +"\n" +"Returns a tuple of (task_info, stack_frames) where:\n" +"- task_info is a tuple of (task_id, task_name) identifying the task\n" +"- stack_frames is a list of tuples (function_name, filename, line_number) representing\n" +" the Python stack frames for the task, ordered from most recent to oldest\n" +"\n" +"Example:\n" +" ((4345585712, \'Task-1\'), [\n" +" (\'run_echo_server\', \'server.py\', 127),\n" +" (\'serve_forever\', \'server.py\', 45),\n" +" (\'main\', \'app.py\', 23)\n" +" ])\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" RuntimeError: If there is an error copying memory from the target process\n" +" OSError: If there is an error accessing the target process\n" +" PermissionError: If access to the target process is denied\n" +" UnicodeDecodeError: If there is an error decoding strings from the target process"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ + {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=654772085f1f4bf6 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 4f06a36a130207..ee869d991d93cd 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -2364,6 +2364,8 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyObject *self) { + PyThreadState *tstate = PyThreadState_GET(); + _Py_atomic_add_uint64(&tstate->interp->_code_object_generation, 1); PyCodeObject *co = _PyCodeObject_CAST(self); _PyObject_ResurrectStart(self); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); diff --git a/Python/index_pool.c b/Python/index_pool.c index 007c81a0fc16ec..520a65938ec6c7 100644 --- a/Python/index_pool.c +++ b/Python/index_pool.c @@ -172,6 +172,9 @@ _PyIndexPool_AllocIndex(_PyIndexPool *pool) else { index = heap_pop(free_indices); } + + pool->tlbc_generation++; + UNLOCK_POOL(pool); return index; } @@ -180,6 +183,7 @@ void _PyIndexPool_FreeIndex(_PyIndexPool *pool, int32_t index) { LOCK_POOL(pool); + pool->tlbc_generation++; heap_add(&pool->free_indices, index); UNLOCK_POOL(pool); } diff --git a/Python/pystate.c b/Python/pystate.c index 14ae2748b0bc99..5aefd03b30ede8 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -676,6 +676,7 @@ init_interpreter(PyInterpreterState *interp, } interp->sys_profile_initialized = false; interp->sys_trace_initialized = false; + interp->_code_object_generation = 0; interp->jit = false; interp->executor_list_head = NULL; interp->executor_deletion_list_head = NULL; @@ -886,6 +887,10 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } + interp->_code_object_generation = 0; +#ifdef Py_GIL_DISABLED + interp->tlbc_indices.tlbc_generation = 0; +#endif PyConfig_Clear(&interp->config); _PyCodec_Fini(interp); @@ -1457,9 +1462,6 @@ tstate_is_alive(PyThreadState *tstate) // lifecycle //---------- -/* Minimum size of data stack chunk */ -#define DATA_STACK_CHUNK_SIZE (16*1024) - static _PyStackChunk* allocate_chunk(int size_in_bytes, _PyStackChunk* previous) { @@ -3012,7 +3014,7 @@ _PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature static PyObject ** push_chunk(PyThreadState *tstate, int size) { - int allocate_size = DATA_STACK_CHUNK_SIZE; + int allocate_size = _PY_DATA_STACK_CHUNK_SIZE; while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) { allocate_size *= 2; } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index edc77c302916ca..dbc6bdd09a693f 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -73,19 +73,71 @@ extern "C" { # define HAVE_PROCESS_VM_READV 0 #endif +static inline size_t +get_page_size(void) { + size_t page_size = 0; + if (page_size == 0) { +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + page_size = si.dwPageSize; +#else + page_size = (size_t)getpagesize(); +#endif + } + return page_size; +} + +typedef struct page_cache_entry { + uintptr_t page_addr; // page-aligned base address + char *data; + int valid; + struct page_cache_entry *next; +} page_cache_entry_t; + +#define MAX_PAGES 1024 + // Define a platform-independent process handle structure typedef struct { pid_t pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) + mach_port_t task; +#elif defined(MS_WINDOWS) HANDLE hProcess; #endif + page_cache_entry_t pages[MAX_PAGES]; + Py_ssize_t page_size; } proc_handle_t; +static void +_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + PyMem_RawFree(handle->pages[i].data); + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } +} + +void +_Py_RemoteDebug_ClearCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].valid = 0; + } +} + +#if defined(__APPLE__) && TARGET_OS_OSX +static mach_port_t pid_to_task(pid_t pid); +#endif + // Initialize the process handle static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->pid = pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) + handle->task = pid_to_task(handle->pid); +#elif defined(MS_WINDOWS) handle->hProcess = OpenProcess( PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION, FALSE, pid); @@ -94,6 +146,11 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { return -1; } #endif + handle->page_size = get_page_size(); + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } return 0; } @@ -107,6 +164,7 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { } #endif handle->pid = 0; + _Py_RemoteDebug_FreePageCache(handle); } #if defined(__APPLE__) && TARGET_OS_OSX @@ -755,7 +813,7 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #elif defined(__APPLE__) && TARGET_OS_OSX Py_ssize_t result = -1; kern_return_t kr = mach_vm_read_overwrite( - pid_to_task(handle->pid), + handle->task, (mach_vm_address_t)remote_address, len, (mach_vm_address_t)dst, @@ -780,6 +838,59 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #endif } +int +_Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, + uintptr_t addr, + size_t size, + void *out) +{ + size_t page_size = handle->page_size; + uintptr_t page_base = addr & ~(page_size - 1); + size_t offset_in_page = addr - page_base; + + if (offset_in_page + size > page_size) { + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); + } + + // Search for valid cached page + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (entry->valid && entry->page_addr == page_base) { + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + + // Find reusable slot + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (!entry->valid) { + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); + if (entry->data == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact ammount as a fallback + PyErr_Clear(); + goto fallback; + } + + entry->page_addr = page_base; + entry->valid = 1; + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + +fallback: + // Cache full — fallback to uncached read + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); +} + static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, From 7216f7b59c30313fa27a30765fc7340df2f5ccef Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 01:33:02 +0200 Subject: [PATCH 240/713] [3.14] gh-69605: Disable PyREPL module autocomplete fallback on regular completion (gh-134181) (gh-134680) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 0e3bc962c6462f836751e35ef35fa837fd952550) Co-authored-by: Loïc Simon --- Lib/_pyrepl/_module_completer.py | 4 +-- Lib/_pyrepl/readline.py | 5 ++-- Lib/test/test_pyrepl/test_pyrepl.py | 28 +++++++++++++------ ...5-05-18-14-33-23.gh-issue-69605.ZMO49F.rst | 2 ++ 4 files changed, 27 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-14-33-23.gh-issue-69605.ZMO49F.rst diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 494a501101a9b2..1e9462a42156d4 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -42,11 +42,11 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self._global_cache: list[pkgutil.ModuleInfo] = [] self._curr_sys_path: list[str] = sys.path[:] - def get_completions(self, line: str) -> list[str]: + def get_completions(self, line: str) -> list[str] | None: """Return the next possible import completions for 'line'.""" result = ImportParser(line).parse() if not result: - return [] + return None try: return self.complete(*result) except Exception: diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 572eee520e53f3..9560ae779abfea 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -134,7 +134,8 @@ def get_stem(self) -> str: return "".join(b[p + 1 : self.pos]) def get_completions(self, stem: str) -> list[str]: - if module_completions := self.get_module_completions(): + module_completions = self.get_module_completions() + if module_completions is not None: return module_completions if len(stem) == 0 and self.more_lines is not None: b = self.buffer @@ -165,7 +166,7 @@ def get_completions(self, stem: str) -> list[str]: result.sort() return result - def get_module_completions(self) -> list[str]: + def get_module_completions(self) -> list[str] | None: line = self.get_line() return self.config.module_completer.get_completions(line) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index abb4bd1bc25fb1..aa3a592766d6d1 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -918,7 +918,14 @@ def test_func(self): class TestPyReplModuleCompleter(TestCase): def setUp(self): + import importlib + # Make iter_modules() search only the standard library. + # This makes the test more reliable in case there are + # other user packages/scripts on PYTHONPATH which can + # interfere with the completions. + lib_path = os.path.dirname(importlib.__path__[0]) self._saved_sys_path = sys.path + sys.path = [lib_path] def tearDown(self): sys.path = self._saved_sys_path @@ -932,14 +939,6 @@ def prepare_reader(self, events, namespace): return reader def test_import_completions(self): - import importlib - # Make iter_modules() search only the standard library. - # This makes the test more reliable in case there are - # other user packages/scripts on PYTHONPATH which can - # intefere with the completions. - lib_path = os.path.dirname(importlib.__path__[0]) - sys.path = [lib_path] - cases = ( ("import path\t\n", "import pathlib"), ("import importlib.\t\tres\t\n", "import importlib.resources"), @@ -1052,6 +1051,19 @@ def test_invalid_identifiers(self): output = reader.readline() self.assertEqual(output, expected) + def test_no_fallback_on_regular_completion(self): + cases = ( + ("import pri\t\n", "import pri"), + ("from pri\t\n", "from pri"), + ("from typing import Na\t\n", "from typing import Na"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-14-33-23.gh-issue-69605.ZMO49F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-14-33-23.gh-issue-69605.ZMO49F.rst new file mode 100644 index 00000000000000..7b7275fee69b9b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-14-33-23.gh-issue-69605.ZMO49F.rst @@ -0,0 +1,2 @@ +When auto-completing an import in the :term:`REPL`, finding no candidates +now issues no suggestion, rather than suggestions from the current namespace. From 16187b58bff9ccf6f800ba908ad5dae441fcb6a1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 05:23:45 +0200 Subject: [PATCH 241/713] [3.14] gh-132876: workaround broken ldexp() on Windows 10 (GH-133135) (#134684) gh-132876: workaround broken ldexp() on Windows 10 (GH-133135) * gh-132876: workaround broken ldexp() on Windows 10 ldexp() fails to round subnormal results before Windows 11, so hide their bug. (cherry picked from commit cf8941c60356acdd00055e5583a2d64761c34af4) Co-authored-by: Sergey B Kirpichev Co-authored-by: Tim Peters --- Lib/test/test_math.py | 6 ++++++ ...-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst | 4 ++++ Modules/mathmodule.c | 21 +++++++++++++++++++ 3 files changed, 31 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 913a60bf9e04e3..d14336f8bac498 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1214,6 +1214,12 @@ def testLdexp(self): self.assertEqual(math.ldexp(NINF, n), NINF) self.assertTrue(math.isnan(math.ldexp(NAN, n))) + @requires_IEEE_754 + def testLdexp_denormal(self): + # Denormal output incorrectly rounded (truncated) + # on some Windows. + self.assertEqual(math.ldexp(6993274598585239, -1126), 1e-323) + def testLog(self): self.assertRaises(TypeError, math.log) self.assertRaises(TypeError, math.log, 1, 2, 3) diff --git a/Misc/NEWS.d/next/Library/2025-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst b/Misc/NEWS.d/next/Library/2025-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst new file mode 100644 index 00000000000000..cb3ca3321e3d26 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst @@ -0,0 +1,4 @@ +``ldexp()`` on Windows doesn't round subnormal results before Windows 11, +but should. Python's :func:`math.ldexp` wrapper now does round them, so +results may change slightly, in rare cases of very small results, on +Windows versions before 11. diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 40abd69f0a6600..71d9c1387f5780 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -2161,6 +2161,27 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); +#ifdef _MSC_VER + if (DBL_MIN > r && r > -DBL_MIN) { + /* Denormal (or zero) results can be incorrectly rounded here (rather, + truncated). Fixed in newer versions of the C runtime, included + with Windows 11. */ + int original_exp; + frexp(x, &original_exp); + if (original_exp > DBL_MIN_EXP) { + /* Shift down to the smallest normal binade. No bits lost. */ + int shift = DBL_MIN_EXP - original_exp; + x = ldexp(x, shift); + exp -= shift; + } + /* Multiplying by 2**exp finishes the job, and the HW will round as + appropriate. Note: if exp < -DBL_MANT_DIG, all of x is shifted + to be < 0.5ULP of smallest denorm, so should be thrown away. If + exp is so very negative that ldexp underflows to 0, that's fine; + no need to check in advance. */ + r = x*ldexp(1.0, (int)exp); + } +#endif if (isinf(r)) errno = ERANGE; } From ebfd18b3f01d00a2246003ea4dd5ad5e4bf6d570 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 10:00:17 +0200 Subject: [PATCH 242/713] [3.14] gh-127833: lexical analysis: Improve section on Names (GH-131474) (#134423) Co-authored-by: Petr Viktorin Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> Co-authored-by: Blaise Pabon --- Doc/reference/lexical_analysis.rst | 128 +++++++++++++++++------------ Tools/unicode/makeunicodedata.py | 2 +- 2 files changed, 77 insertions(+), 53 deletions(-) diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 001e2547fe8031..6c4a4ea81afe29 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -288,58 +288,81 @@ forms a legal token, when read from left to right. .. _identifiers: -Identifiers and keywords -======================== +Names (identifiers and keywords) +================================ .. index:: identifier, name -Identifiers (also referred to as *names*) are described by the following lexical -definitions. +:data:`~token.NAME` tokens represent *identifiers*, *keywords*, and +*soft keywords*. -The syntax of identifiers in Python is based on the Unicode standard annex -UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for -further details. - -Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -include the uppercase and lowercase letters ``A`` through -``Z``, the underscore ``_`` and, except for the first character, the digits +Within the ASCII range (U+0001..U+007F), the valid characters for names +include the uppercase and lowercase letters (``A-Z`` and ``a-z``), +the underscore ``_`` and, except for the first character, the digits ``0`` through ``9``. -Python 3.0 introduced additional characters from outside the ASCII range (see -:pep:`3131`). For these characters, the classification uses the version of the -Unicode Character Database as included in the :mod:`unicodedata` module. -Identifiers are unlimited in length. Case is significant. +Names must contain at least one character, but have no upper length limit. +Case is significant. -.. productionlist:: python-grammar - identifier: `xid_start` `xid_continue`* - id_start: - id_continue: - xid_start: - xid_continue: - -The Unicode category codes mentioned above stand for: - -* *Lu* - uppercase letters -* *Ll* - lowercase letters -* *Lt* - titlecase letters -* *Lm* - modifier letters -* *Lo* - other letters -* *Nl* - letter numbers -* *Mn* - nonspacing marks -* *Mc* - spacing combining marks -* *Nd* - decimal numbers -* *Pc* - connector punctuations -* *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards - compatibility -* *Other_ID_Continue* - likewise - -All identifiers are converted into the normal form NFKC while parsing; comparison -of identifiers is based on NFKC. - -A non-normative HTML file listing all valid identifier characters for Unicode -16.0.0 can be found at -https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +Besides ``A-Z``, ``a-z``, ``_`` and ``0-9``, names can also use "letter-like" +and "number-like" characters from outside the ASCII range, as detailed below. + +All identifiers are converted into the `normalization form`_ NFKC while +parsing; comparison of identifiers is based on NFKC. + +Formally, the first character of a normalized identifier must belong to the +set ``id_start``, which is the union of: + +* Unicode category ```` - uppercase letters (includes ``A`` to ``Z``) +* Unicode category ```` - lowercase letters (includes ``a`` to ``z``) +* Unicode category ```` - titlecase letters +* Unicode category ```` - modifier letters +* Unicode category ```` - other letters +* Unicode category ```` - letter numbers +* {``"_"``} - the underscore +* ```` - an explicit set of characters in `PropList.txt`_ + to support backwards compatibility + +The remaining characters must belong to the set ``id_continue``, which is the +union of: + +* all characters in ``id_start`` +* Unicode category ```` - decimal numbers (includes ``0`` to ``9``) +* Unicode category ```` - connector punctuations +* Unicode category ```` - nonspacing marks +* Unicode category ```` - spacing combining marks +* ```` - another explicit set of characters in + `PropList.txt`_ to support backwards compatibility + +Unicode categories use the version of the Unicode Character Database as +included in the :mod:`unicodedata` module. + +These sets are based on the Unicode standard annex `UAX-31`_. +See also :pep:`3131` for further details. + +Even more formally, names are described by the following lexical definitions: + +.. grammar-snippet:: + :group: python-grammar + + NAME: `xid_start` `xid_continue`* + id_start: | | | | | | "_" | + id_continue: `id_start` | | | | | + xid_start: + xid_continue: + identifier: <`NAME`, except keywords> + +A non-normative listing of all valid identifier characters as defined by +Unicode is available in the `DerivedCoreProperties.txt`_ file in the Unicode +Character Database. + + +.. _UAX-31: https://www.unicode.org/reports/tr31/ +.. _PropList.txt: https://www.unicode.org/Public/16.0.0/ucd/PropList.txt +.. _DerivedCoreProperties.txt: https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +.. _normalization form: https://www.unicode.org/reports/tr15/#Norm_Forms .. _keywords: @@ -351,7 +374,7 @@ Keywords single: keyword single: reserved word -The following identifiers are used as reserved words, or *keywords* of the +The following names are used as reserved words, or *keywords* of the language, and cannot be used as ordinary identifiers. They must be spelled exactly as written here: @@ -375,18 +398,19 @@ Soft Keywords .. versionadded:: 3.10 -Some identifiers are only reserved under specific contexts. These are known as -*soft keywords*. The identifiers ``match``, ``case``, ``type`` and ``_`` can -syntactically act as keywords in certain contexts, +Some names are only reserved under specific contexts. These are known as +*soft keywords*: + +- ``match``, ``case``, and ``_``, when used in the :keyword:`match` statement. +- ``type``, when used in the :keyword:`type` statement. + +These syntactically act as keywords in their specific contexts, but this distinction is done at the parser level, not when tokenizing. As soft keywords, their use in the grammar is possible while still preserving compatibility with existing code that uses these names as identifier names. -``match``, ``case``, and ``_`` are used in the :keyword:`match` statement. -``type`` is used in the :keyword:`type` statement. - .. versionchanged:: 3.12 ``type`` is now a soft keyword. diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 889ae8fc869b8a..d4cca68c3e3e71 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -43,7 +43,7 @@ # When changing UCD version please update # * Doc/library/stdtypes.rst, and # * Doc/library/unicodedata.rst -# * Doc/reference/lexical_analysis.rst (two occurrences) +# * Doc/reference/lexical_analysis.rst (three occurrences) UNIDATA_VERSION = "16.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" From 1ce68f6297560c3d22169f0280a516b79375e947 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 10:02:52 +0200 Subject: [PATCH 243/713] [3.14] gh-134152: Fix UnboundLocalError in email._header_value_parser _get_ptext_to_endchars (GH-134233) (#134678) Co-authored-by: R. David Murray --- Lib/email/_header_value_parser.py | 2 ++ .../test_email/test__header_value_parser.py | 33 +++++++++++++++++++ ...-05-19-10-32-11.gh-issue-134152.INJC2j.rst | 2 ++ 3 files changed, 37 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-19-10-32-11.gh-issue-134152.INJC2j.rst diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9a51b9437333db..f11fa83d45ed2d 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1020,6 +1020,8 @@ def _get_ptext_to_endchars(value, endchars): a flag that is True iff there were any quoted printables decoded. """ + if not value: + return '', '', False fragment, *remainder = _wsp_splitter(value, 1) vchars = [] escape = False diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index ac12c3b2306f7d..fd4ac2c404ce47 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -463,6 +463,19 @@ def test_get_qp_ctext_non_printables(self): [errors.NonPrintableDefect], ')') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qp_ctext_close_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + ')', '', ' ', [], ')') + + def test_get_qp_ctext_open_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + '(', '', ' ', [], '(') + + def test_get_qp_ctext_no_end_char(self): + self._test_get_x(parser.get_qp_ctext, + '', '', ' ', [], '') + + # get_qcontent def test_get_qcontent_only(self): @@ -503,6 +516,14 @@ def test_get_qcontent_non_printables(self): [errors.NonPrintableDefect], '"') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qcontent_empty(self): + self._test_get_x(parser.get_qcontent, + '"', '', '', [], '"') + + def test_get_qcontent_no_end_char(self): + self._test_get_x(parser.get_qcontent, + '', '', '', [], '') + # get_atext def test_get_atext_only(self): @@ -1283,6 +1304,18 @@ def test_get_dtext_open_bracket_mid_word(self): self._test_get_x(parser.get_dtext, 'foo[bar', 'foo', 'foo', [], '[bar') + def test_get_dtext_open_bracket_only(self): + self._test_get_x(parser.get_dtext, + '[', '', '', [], '[') + + def test_get_dtext_close_bracket_only(self): + self._test_get_x(parser.get_dtext, + ']', '', '', [], ']') + + def test_get_dtext_empty(self): + self._test_get_x(parser.get_dtext, + '', '', '', [], '') + # get_domain_literal def test_get_domain_literal_only(self): diff --git a/Misc/NEWS.d/next/Library/2025-05-19-10-32-11.gh-issue-134152.INJC2j.rst b/Misc/NEWS.d/next/Library/2025-05-19-10-32-11.gh-issue-134152.INJC2j.rst new file mode 100644 index 00000000000000..6da3d4147dd960 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-19-10-32-11.gh-issue-134152.INJC2j.rst @@ -0,0 +1,2 @@ +Fixed :exc:`UnboundLocalError` that could occur during :mod:`email` header +parsing if an expected trailing delimiter is missing in some contexts. From dc441efbd00b8dd2f26d014fb67895d381a8f602 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 10:22:14 +0200 Subject: [PATCH 244/713] [3.14] gh-132493: Remove __annotations__ usage in inspect._signature_is_functionlike (GH-133415) (#133796) gh-132493: Remove __annotations__ usage in inspect._signature_is_functionlike (GH-133415) This check is potentially problematic because it could force evaluation of annotations unnecessarily. This doesn't trigger for builtin objects (functions, classes, or modules) with annotations, but it could trigger for third-party objects. The check was not particularly useful anyway, because it succeeds if ``__annotations__`` is a dict or None, so the only thing this did was guard against objects that have an ``__annotations__`` attribute that is of some other type. That doesn't seem particularly useful, so I just removed the check. (cherry picked from commit cb6596c6aa30c8c3213c8746d5590e6e2757a6c1) Co-authored-by: Jelle Zijlstra --- Lib/inspect.py | 4 +-- Lib/test/test_inspect/test_inspect.py | 31 +++++++++++++++++++ ...-05-04-17-04-55.gh-issue-132493.huirKi.rst | 2 ++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-04-17-04-55.gh-issue-132493.huirKi.rst diff --git a/Lib/inspect.py b/Lib/inspect.py index 52c9bb05b31f37..183e67fabf966e 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2074,13 +2074,11 @@ def _signature_is_functionlike(obj): code = getattr(obj, '__code__', None) defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here - annotations = getattr(obj, '__annotations__', None) return (isinstance(code, types.CodeType) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and - (kwdefaults is None or isinstance(kwdefaults, dict)) and - (isinstance(annotations, (dict)) or annotations is None) ) + (kwdefaults is None or isinstance(kwdefaults, dict))) def _signature_strip_non_python_syntax(signature): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 57dc7a24af94b8..44f7a54bbf1c1e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -4997,6 +4997,37 @@ def test_signature_annotation_format(self): with self.assertRaisesRegex(NameError, "undefined"): signature_func(ida.f) + def test_signature_deferred_annotations(self): + def f(x: undef): + pass + + class C: + x: undef + + def __init__(self, x: undef): + self.x = x + + sig = inspect.signature(f, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + sig = inspect.signature(C, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + + class CallableWrapper: + def __init__(self, func): + self.func = func + self.__annotate__ = func.__annotate__ + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __annotations__(self): + return self.__annotate__(Format.VALUE) + + cw = CallableWrapper(f) + sig = inspect.signature(cw, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['args', 'kwargs']) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct diff --git a/Misc/NEWS.d/next/Library/2025-05-04-17-04-55.gh-issue-132493.huirKi.rst b/Misc/NEWS.d/next/Library/2025-05-04-17-04-55.gh-issue-132493.huirKi.rst new file mode 100644 index 00000000000000..ad06ee6b7a2757 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-04-17-04-55.gh-issue-132493.huirKi.rst @@ -0,0 +1,2 @@ +Avoid accessing ``__annotations__`` unnecessarily in +:func:`inspect.signature`. From ed52549c3716f5aa5942db2a594ba594edba2e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 26 May 2025 12:09:02 +0200 Subject: [PATCH 245/713] [3.14] gh-134208: remove dead AC directives for `_curses.window.{chgat,getstr,instr}` (GH-134325) (#134701) (cherry picked from commit 29e81159644cf78d958e30aaef208e22a04a8b25) --- Modules/_cursesmodule.c | 321 ++++++++++++++++++---------------------- 1 file changed, 148 insertions(+), 173 deletions(-) diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index f6e366ef7dec8b..806153fc16ccf1 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -1312,32 +1312,27 @@ int py_mvwdelch(WINDOW *w, int y, int x) /* chgat, added by Fabian Kreutz */ #ifdef HAVE_CURSES_WCHGAT -/*[-clinic input] -_curses.window.chgat - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] +PyDoc_STRVAR(_curses_window_chgat__doc__, +"chgat([y, x,] [n=-1,] attr)\n" +"Set the attributes of characters.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Number of characters.\n" +" attr\n" +" Attributes for characters.\n" +"\n" +"Set the attributes of num characters at the current cursor position, or at\n" +"position (y, x) if supplied. If no value of num is given or num = -1, the\n" +"attribute will be set on all the characters to the end of the line. This\n" +"function does not move the cursor. The changed line will be touched using\n" +"the touchline() method so that the contents will be redisplayed by the next\n" +"window refresh."); - n: int = -1 - Number of characters. - - attr: long - Attributes for characters. - / - -Set the attributes of characters. - -Set the attributes of num characters at the current cursor position, or at -position (y, x) if supplied. If no value of num is given or num = -1, the -attribute will be set on all the characters to the end of the line. This -function does not move the cursor. The changed line will be touched using -the touchline() method so that the contents will be redisplayed by the next -window refresh. -[-clinic start generated code]*/ static PyObject * PyCursesWindow_ChgAt(PyObject *op, PyObject *args) { @@ -1363,19 +1358,20 @@ PyCursesWindow_ChgAt(PyObject *op, PyObject *args) attr = lattr; break; case 3: - if (!PyArg_ParseTuple(args,"iil;int,int,attr", &y, &x, &lattr)) + if (!PyArg_ParseTuple(args,"iil;y,x,attr", &y, &x, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; case 4: - if (!PyArg_ParseTuple(args,"iiil;int,int,n,attr", &y, &x, &num, &lattr)) + if (!PyArg_ParseTuple(args,"iiil;y,x,n,attr", &y, &x, &num, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; default: - PyErr_SetString(PyExc_TypeError, "chgat requires 1 to 4 arguments"); + PyErr_SetString(PyExc_TypeError, + "_curses.window.chgat requires 1 to 4 arguments"); return NULL; } @@ -1678,102 +1674,102 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, } #endif -/*[-clinic input] -_curses.window.getstr - - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 2047 - Maximal number of characters. - / +/* + * Helper function for parsing parameters from getstr() and instr(). + * This function is necessary because Argument Clinic does not know + * how to handle nested optional groups with default values inside. + * + * Return 1 on success and 0 on failure, similar to PyArg_ParseTuple(). + */ +static int +curses_clinic_parse_optional_xy_n(PyObject *args, + int *y, int *x, unsigned int *n, int *use_xy, + const char *qualname) +{ + switch (PyTuple_GET_SIZE(args)) { + case 0: { + *use_xy = 0; + return 1; + } + case 1: { + *use_xy = 0; + return PyArg_ParseTuple(args, "O&;n", + _PyLong_UnsignedInt_Converter, n); + } + case 2: { + *use_xy = 1; + return PyArg_ParseTuple(args, "ii;y,x", y, x); + } + case 3: { + *use_xy = 1; + return PyArg_ParseTuple(args, "iiO&;y,x,n", y, x, + _PyLong_UnsignedInt_Converter, n); + } + default: { + *use_xy = 0; + PyErr_Format(PyExc_TypeError, "%s requires 0 to 3 arguments", + qualname); + return 0; + } + } +} -Read a string from the user, with primitive line editing capacity. -[-clinic start generated code]*/ +PyDoc_STRVAR(_curses_window_getstr__doc__, +"getstr([[y, x,] n=2047])\n" +"Read a string from the user, with primitive line editing capacity.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters."); static PyObject * -PyCursesWindow_GetStr(PyObject *op, PyObject *args) +PyCursesWindow_getstr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + PyObject *res; - int x, y, n; - int rtn; + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { + return NULL; + } - /* could make the buffer size larger/dynamic */ - Py_ssize_t max_buf_size = 2048; - PyObject *result = PyBytes_FromStringAndSize(NULL, max_buf_size); - if (result == NULL) + n = Py_MIN(n, max_buf_size - 1); + res = PyBytes_FromStringAndSize(NULL, n + 1); + if (res == NULL) { return NULL; - char *buf = PyBytes_AS_STRING(result); + } + char *buf = PyBytes_AS_STRING(res); - switch (PyTuple_Size(args)) { - case 0: - Py_BEGIN_ALLOW_THREADS - rtn = wgetnstr(self->win, buf, max_buf_size - 1); - Py_END_ALLOW_THREADS - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - goto error; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - goto error; - } - Py_BEGIN_ALLOW_THREADS - rtn = wgetnstr(self->win, buf, Py_MIN(n, max_buf_size - 1)); - Py_END_ALLOW_THREADS - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - goto error; + if (use_xy) { Py_BEGIN_ALLOW_THREADS #ifdef STRICT_SYSV_CURSES - rtn = wmove(self->win,y,x)==ERR ? ERR : wgetnstr(self->win, rtn, max_buf_size - 1); + rtn = wmove(self->win, y, x) == ERR + ? ERR + : wgetnstr(self->win, buf, n); #else - rtn = mvwgetnstr(self->win,y,x,buf, max_buf_size - 1); + rtn = mvwgetnstr(self->win, y, x, buf, n); #endif Py_END_ALLOW_THREADS - break; - case 3: - if (!PyArg_ParseTuple(args,"iii;y,x,n", &y, &x, &n)) - goto error; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - goto error; - } -#ifdef STRICT_SYSV_CURSES - Py_BEGIN_ALLOW_THREADS - rtn = wmove(self->win,y,x)==ERR ? ERR : - wgetnstr(self->win, rtn, Py_MIN(n, max_buf_size - 1)); - Py_END_ALLOW_THREADS -#else + } + else { Py_BEGIN_ALLOW_THREADS - rtn = mvwgetnstr(self->win, y, x, buf, Py_MIN(n, max_buf_size - 1)); + rtn = wgetnstr(self->win, buf, n); Py_END_ALLOW_THREADS -#endif - break; - default: - PyErr_SetString(PyExc_TypeError, "getstr requires 0 to 3 arguments"); - goto error; } if (rtn == ERR) { - Py_DECREF(result); + Py_DECREF(res); return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - - if (_PyBytes_Resize(&result, strlen(buf)) < 0) { - return NULL; - } - - return result; - -error: - Py_DECREF(result); - return NULL; + _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure + return res; } /*[clinic input] @@ -1898,87 +1894,57 @@ _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, return rtn; } -/*[-clinic input] -_curses.window.instr - - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 2047 - Maximal number of characters. - / +PyDoc_STRVAR(_curses_window_instr__doc__, +"instr([y, x,] n=2047)\n" +"Return a string of characters, extracted from the window.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters.\n" +"\n" +"Return a string of characters, extracted from the window starting at the\n" +"current cursor position, or at y, x if specified. Attributes are stripped\n" +"from the characters. If n is specified, instr() returns a string at most\n" +"n characters long (exclusive of the trailing NUL)."); -Return a string of characters, extracted from the window. - -Return a string of characters, extracted from the window starting at the -current cursor position, or at y, x if specified. Attributes are stripped -from the characters. If n is specified, instr() returns a string at most -n characters long (exclusive of the trailing NUL). -[-clinic start generated code]*/ static PyObject * -PyCursesWindow_InStr(PyObject *op, PyObject *args) +PyCursesWindow_instr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + PyObject *res; - int x, y, n; - int rtn; + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { + return NULL; + } - /* could make the buffer size larger/dynamic */ - Py_ssize_t max_buf_size = 2048; - PyObject *result = PyBytes_FromStringAndSize(NULL, max_buf_size); - if (result == NULL) + n = Py_MIN(n, max_buf_size - 1); + res = PyBytes_FromStringAndSize(NULL, n + 1); + if (res == NULL) { return NULL; - char *buf = PyBytes_AS_STRING(result); + } + char *buf = PyBytes_AS_STRING(res); - switch (PyTuple_Size(args)) { - case 0: - rtn = winnstr(self->win, buf, max_buf_size - 1); - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - goto error; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - goto error; - } - rtn = winnstr(self->win, buf, Py_MIN(n, max_buf_size - 1)); - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - goto error; - rtn = mvwinnstr(self->win, y, x, buf, max_buf_size - 1); - break; - case 3: - if (!PyArg_ParseTuple(args, "iii;y,x,n", &y, &x, &n)) - goto error; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - goto error; - } - rtn = mvwinnstr(self->win, y, x, buf, Py_MIN(n, max_buf_size - 1)); - break; - default: - PyErr_SetString(PyExc_TypeError, "instr requires 0 or 3 arguments"); - goto error; + if (use_xy) { + rtn = mvwinnstr(self->win, y, x, buf, n); + } + else { + rtn = winnstr(self->win, buf, n); } if (rtn == ERR) { - Py_DECREF(result); + Py_DECREF(res); return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } - - if (_PyBytes_Resize(&result, strlen(buf)) < 0) { - return NULL; - } - - return result; - -error: - Py_DECREF(result); - return NULL; + _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure + return res; } /*[clinic input] @@ -2689,7 +2655,10 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ATTRSET_METHODDEF _CURSES_WINDOW_BKGD_METHODDEF #ifdef HAVE_CURSES_WCHGAT - {"chgat", PyCursesWindow_ChgAt, METH_VARARGS}, + { + "chgat", PyCursesWindow_ChgAt, METH_VARARGS, + _curses_window_chgat__doc__ + }, #endif _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF @@ -2712,7 +2681,10 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_GET_WCH_METHODDEF {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, - {"getstr", PyCursesWindow_GetStr, METH_VARARGS}, + { + "getstr", PyCursesWindow_getstr, METH_VARARGS, + _curses_window_getstr__doc__ + }, {"getyx", PyCursesWindow_getyx, METH_NOARGS}, _CURSES_WINDOW_HLINE_METHODDEF {"idcok", PyCursesWindow_idcok, METH_VARARGS}, @@ -2726,7 +2698,10 @@ static PyMethodDef PyCursesWindow_methods[] = { {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF - {"instr", PyCursesWindow_InStr, METH_VARARGS}, + { + "instr", PyCursesWindow_instr, METH_VARARGS, + _curses_window_instr__doc__ + }, _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, {"keypad", PyCursesWindow_keypad, METH_VARARGS}, From c8379c7edf8d2d9e8bb60c343470429809ee067b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 12:29:51 +0200 Subject: [PATCH 246/713] [3.14] gh-134559: Add versionadded for ``object.__replace__()`` (GH-134672) (#134699) gh-134559: Add versionadded for ``object.__replace__()`` (GH-134672) (cherry picked from commit 71290a6fbe79cef3773a3af6fcf08d8bf15c8460) Co-authored-by: tmlnv <108088921+tmlnv@users.noreply.github.com> --- Doc/library/copy.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 95b41f988a035b..210ad7188003e6 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -122,6 +122,8 @@ and only supports named tuples created by :func:`~collections.namedtuple`, This method should create a new object of the same type, replacing fields with values from *changes*. + .. versionadded:: 3.13 + .. seealso:: From 072f50fc12ee7e11e84f6d85d8a1b3ad2c65db9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Mon, 26 May 2025 12:45:36 +0200 Subject: [PATCH 247/713] [3.14] gh-132710: only use stable `_uuid.generate_time_safe()` to deduce MAC address (GH-132901) (#134697) (cherry picked from commit 3bffada46728e51f84c41ecbb0d3722595693e63) --- Lib/test/test_uuid.py | 28 ++++++ Lib/uuid.py | 6 +- Modules/_uuidmodule.c | 40 ++++++-- configure | 209 ++++++++++++++++++++++++++++++++++++++++-- configure.ac | 99 ++++++++++++++++++-- pyconfig.h.in | 3 + 6 files changed, 357 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 958be5408ce90a..7ddacf07a2c789 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -14,6 +14,7 @@ from test import support from test.support import import_helper +from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -1217,10 +1218,37 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): uuid = py_uuid + @unittest.skipUnless(c_uuid, 'requires the C _uuid module') class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/uuid.py b/Lib/uuid.py index 036ffebf67a0be..06f81a7c338372 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -633,22 +633,24 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = _uuid.has_stable_extractable_node _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index c5e78b1510b5e3..c31a7e8fea5608 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -78,23 +78,47 @@ py_UuidCreate(PyObject *Py_UNUSED(context), return NULL; } +static int +py_windows_has_stable_node(void) +{ + UUID uuid; + RPC_STATUS res; + Py_BEGIN_ALLOW_THREADS + res = UuidCreateSequential(&uuid); + Py_END_ALLOW_THREADS + return res == RPC_S_OK; +} #endif /* MS_WINDOWS */ static int -uuid_exec(PyObject *module) { +uuid_exec(PyObject *module) +{ +#define ADD_INT(NAME, VALUE) \ + do { \ + if (PyModule_AddIntConstant(module, (NAME), (VALUE)) < 0) { \ + return -1; \ + } \ + } while (0) + assert(sizeof(uuid_t) == 16); #if defined(MS_WINDOWS) - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #elif defined(HAVE_UUID_GENERATE_TIME_SAFE) - int has_uuid_generate_time_safe = 1; + ADD_INT("has_uuid_generate_time_safe", 1); #else - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #endif - if (PyModule_AddIntConstant(module, "has_uuid_generate_time_safe", - has_uuid_generate_time_safe) < 0) { - return -1; - } + +#if defined(MS_WINDOWS) + ADD_INT("has_stable_extractable_node", py_windows_has_stable_node()); +#elif defined(HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC) + ADD_INT("has_stable_extractable_node", 1); +#else + ADD_INT("has_stable_extractable_node", 0); +#endif + +#undef ADD_INT return 0; } diff --git a/configure b/configure index 5d21d06014a238..884f8a4b0680d9 100755 --- a/configure +++ b/configure @@ -14052,6 +14052,7 @@ fi + have_uuid=missing for ac_header in uuid.h @@ -14061,6 +14062,7 @@ if test "x$ac_cv_header_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + for ac_func in uuid_create uuid_enc_be do : as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` @@ -14070,7 +14072,9 @@ then : cat >>confdefs.h <<_ACEOF #define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF - have_uuid=yes + + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14160,6 +14164,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14257,8 +14262,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14302,6 +14308,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14399,8 +14406,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14431,10 +14439,16 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } have_uuid=yes - printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h - - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h - + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be *or* . + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include ' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes fi @@ -14455,6 +14469,7 @@ if test "x$ac_cv_func_uuid_generate_time" = xyes then : have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14465,6 +14480,24 @@ fi done +fi + +if test "x$ac_cv_have_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_generate_time_safe" = xyes +then : + + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + fi # gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. @@ -14480,6 +14513,164 @@ then : have_uuid=no fi +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. + + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if uuid_generate_time_safe() node value is stable" >&5 +printf %s "checking if uuid_generate_time_safe() node value is stable... " >&6; } + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include // PRIu64 + #include // uint64_t + #include // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include + #else + #include + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node1=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include // PRIu64 + #include // uint64_t + #include // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include + #else + #include + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node2=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC 1" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: stable" >&5 +printf "%s\n" "stable" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unstable" >&5 +printf "%s\n" "unstable" >&6; } + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux diff --git a/configure.ac b/configure.ac index 8613aa58257f5b..cf25148bad2077 100644 --- a/configure.ac +++ b/configure.ac @@ -3740,15 +3740,17 @@ dnl check for uuid dependencies AH_TEMPLATE([HAVE_UUID_H], [Define to 1 if you have the header file.]) AH_TEMPLATE([HAVE_UUID_UUID_H], [Define to 1 if you have the header file.]) AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE], [Define if uuid_generate_time_safe() exists.]) +AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [Define if uuid_generate_time_safe() is able to deduce a MAC address.]) have_uuid=missing dnl AIX provides support for RFC4122 (uuid) in libc.a starting with AIX 6.1 dnl (anno 2007). FreeBSD and OpenBSD provides support in libc as well. dnl Little-endian FreeBSD, OpenBSD and NetBSD needs encoding into an octet dnl stream in big-endian byte-order -AC_CHECK_HEADERS([uuid.h], - [AC_CHECK_FUNCS([uuid_create uuid_enc_be], - [have_uuid=yes +AC_CHECK_HEADERS([uuid.h], [ + AC_CHECK_FUNCS([uuid_create uuid_enc_be], [ + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) @@ -3758,19 +3760,29 @@ AS_VAR_IF([have_uuid], [missing], [ PKG_CHECK_MODULES( [LIBUUID], [uuid >= 2.20], [dnl linux-util's libuuid has uuid_generate_time_safe() since v2.20 (2011) - dnl and provides . + dnl and provides assuming specific include paths are given have_uuid=yes - AC_DEFINE([HAVE_UUID_H], [1]) - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be *or* . + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include ' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" LIBS="$LIBS $LIBUUID_LIBS" AC_CHECK_HEADERS([uuid/uuid.h], [ + ac_cv_have_uuid_uuid_h=yes PY_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes]) - PY_CHECK_LIB([uuid], [uuid_generate_time_safe], - [have_uuid=yes - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) ]) ]) + PY_CHECK_LIB([uuid], [uuid_generate_time_safe], [ + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes + ])]) AS_VAR_IF([have_uuid], [yes], [ LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-"-luuid"} @@ -3785,12 +3797,19 @@ AS_VAR_IF([have_uuid], [missing], [ AC_CHECK_HEADERS([uuid/uuid.h], [ AC_CHECK_FUNC([uuid_generate_time], [ have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) ]) ]) +AS_VAR_IF([ac_cv_have_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_generate_time_safe], [yes], [ + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) +]) + # gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. # This restriction inhibits the proper generation of time-based UUIDs. if test "$ac_sys_system" = "NetBSD"; then @@ -3800,6 +3819,68 @@ fi AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. +AC_DEFUN([PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC], [WITH_SAVE_ENV([ + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + #include // PRIu64 + #include // uint64_t + #include // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include + #else + #include + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + }]])], [ + AS_VAR_SET([$1], [`cat conftest.out`]) + ], [], [] + )])]) + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + AC_MSG_CHECKING([if uuid_generate_time_safe() node value is stable]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node1]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node2]) + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [1]) + AC_MSG_RESULT([stable]) + else + AC_MSG_RESULT([unstable]) + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux diff --git a/pyconfig.h.in b/pyconfig.h.in index c91facbedf94e5..3dbbda157df70e 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1584,6 +1584,9 @@ /* Define if uuid_generate_time_safe() exists. */ #undef HAVE_UUID_GENERATE_TIME_SAFE +/* Define if uuid_generate_time_safe() is able to deduce a MAC address. */ +#undef HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC + /* Define to 1 if you have the header file. */ #undef HAVE_UUID_H From da3d55403a99a536e5089f8dce435ad2ac8633da Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 12:46:29 +0200 Subject: [PATCH 248/713] [3.14] gh-132710: add missing NEWS entry for GH-134697 (GH-134705) (#134707) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-132710: add missing NEWS entry for GH-134697 (GH-134705) (cherry picked from commit 9eb84d83e00070cec3cfe78f1d0c7a7a0fbef30f) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst diff --git a/Misc/NEWS.d/next/Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst b/Misc/NEWS.d/next/Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst new file mode 100644 index 00000000000000..b7011517aa9db8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst @@ -0,0 +1,3 @@ +If possible, ensure that :func:`uuid.getnode` returns the same result even +across different processes. Previously, the result was constant only within +the same process. Patch by Bénédikt Tran. From 6c917cb11c3b38b6435c12519153f8323995edf7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 26 May 2025 15:10:37 +0200 Subject: [PATCH 249/713] [3.14] gh-91048: Correct Apple platform includes for iOS. (GH-134712) (#134714) Correct Apple platform includes for iOS. (cherry picked from commit 965662ee4a986605b60da470d9e7c1e9a6f922b3) Co-authored-by: Russell Keith-Magee --- Python/remote_debug.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Python/remote_debug.h b/Python/remote_debug.h index dbc6bdd09a693f..6566979bcd0e3e 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -35,7 +35,7 @@ extern "C" { # include #endif -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX # include # include # include @@ -100,7 +100,7 @@ typedef struct page_cache_entry { // Define a platform-independent process handle structure typedef struct { pid_t pid; -#if defined(__APPLE__) +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX mach_port_t task; #elif defined(MS_WINDOWS) HANDLE hProcess; @@ -127,7 +127,7 @@ _Py_RemoteDebug_ClearCache(proc_handle_t *handle) } } -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static mach_port_t pid_to_task(pid_t pid); #endif @@ -135,7 +135,7 @@ static mach_port_t pid_to_task(pid_t pid); static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->pid = pid; -#if defined(__APPLE__) +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX handle->task = pid_to_task(handle->pid); #elif defined(MS_WINDOWS) handle->hProcess = OpenProcess( @@ -167,7 +167,7 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { _Py_RemoteDebug_FreePageCache(handle); } -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static uintptr_t return_section_address64( @@ -481,7 +481,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s return 0; } -#endif // (__APPLE__ && TARGET_OS_OSX) +#endif // (__APPLE__ && defined(TARGET_OS_OSX) && TARGET_OS_OSX) #if defined(__linux__) && HAVE_PROCESS_VM_READV static uintptr_t @@ -759,7 +759,7 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process."); _PyErr_ChainExceptions1(exc); } -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python address = search_map_for_section(handle, "PyRuntime", "libpython"); if (address == 0) { @@ -810,7 +810,7 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX Py_ssize_t result = -1; kern_return_t kr = mach_vm_read_overwrite( handle->task, From 12d3f883ae8aa6297a575511b14fa1e82248427d Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Mon, 26 May 2025 16:26:37 +0300 Subject: [PATCH 250/713] Python 3.14.0b2 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 301 ++++--- Misc/NEWS.d/3.14.0b2.rst | 766 ++++++++++++++++++ ...-04-16-09-38-48.gh-issue-117088.EFt_5c.rst | 1 - ...-04-30-10-22-08.gh-issue-131769.H0oy5x.rst | 1 - ...-05-21-19-46-28.gh-issue-134455.vdwlrq.rst | 2 - ...-05-17-14-41-21.gh-issue-134144.xVpZik.rst | 1 - ...-04-19-17-16-46.gh-issue-132542.7T_TY_.rst | 2 - ...-05-06-15-01-41.gh-issue-133516.RqWVf2.rst | 2 - ...-05-07-23-26-53.gh-issue-133541.bHIC55.rst | 2 - ...-05-08-13-48-02.gh-issue-132762.tKbygC.rst | 1 - ...-05-09-18-11-21.gh-issue-133778.pWEV3t.rst | 2 - ...-05-10-17-12-27.gh-issue-133703.bVM-re.rst | 1 - ...-05-11-13-40-42.gh-issue-133886.ryBAyo.rst | 2 - ...-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst | 2 - ...-05-16-17-25-52.gh-issue-134100.5-FbLK.rst | 2 - ...-05-16-20-59-12.gh-issue-134119.w8expI.rst | 2 - ...-05-17-20-44-51.gh-issue-134158.ewLNLp.rst | 1 - ...5-05-18-14-33-23.gh-issue-69605.ZMO49F.rst | 2 - ...-05-20-14-41-50.gh-issue-128066.qzzGfv.rst | 3 - ...-05-21-15-14-32.gh-issue-130397.aG6EON.rst | 3 - ...-05-21-18-02-56.gh-issue-127960.W3J_2X.rst | 3 - ...-05-22-14-48-19.gh-issue-134381.2BXhth.rst | 1 - .../2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst | 1 - ...2-07-24-20-56-32.gh-issue-69426.unccw7.rst | 3 - ...3-02-13-21-41-34.gh-issue-86155.ppIGSC.rst | 2 - ...3-02-13-21-56-38.gh-issue-62824.CBZzX3.rst | 1 - ...-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst | 1 - ...5-03-30-16-42-38.gh-issue-91555.ShVtwW.rst | 2 - ...-04-26-15-50-12.gh-issue-133009.etBuz5.rst | 3 - ...-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst | 4 - ...-05-04-17-04-55.gh-issue-132493.huirKi.rst | 2 - ...-05-05-22-11-24.gh-issue-133439.LpmyFz.rst | 2 - ...-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst | 2 - ...-05-07-19-16-41.gh-issue-133581.kERUCJ.rst | 4 - ...-05-09-08-49-03.gh-issue-133701.KI8tGz.rst | 3 - ...-05-09-09-10-34.gh-issue-130328.s9h4By.rst | 2 - ...5-05-09-15-50-00.gh-issue-77057.fV8SU-.rst | 2 - ...-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst | 3 - ...-05-09-19-05-24.gh-issue-133783.1voCnR.rst | 3 - ...-05-09-20-59-24.gh-issue-132641.3qTw44.rst | 1 - ...-05-10-12-06-55.gh-issue-133653.Gb2aG4.rst | 7 - ...-05-12-06-52-10.gh-issue-133925.elInBY.rst | 1 - ...-05-12-20-38-57.gh-issue-133960.Aee79f.rst | 3 - ...5-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst | 3 - ...-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst | 2 - ...-05-15-14-27-01.gh-issue-134062.fRbJet.rst | 3 - ...-05-16-12-40-37.gh-issue-132124.T_5Odx.rst | 6 - ...-05-16-20-10-25.gh-issue-134098.YyTkKr.rst | 2 - ...-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst | 3 - ...-05-17-13-46-20.gh-issue-134097.fgkjE1.rst | 1 - ...-05-17-18-08-35.gh-issue-133890.onn9_X.rst | 2 - ...-05-17-20-23-57.gh-issue-133982.smS7au.rst | 3 - ...5-05-18-12-48-39.gh-issue-62184.y11l10.rst | 2 - ...-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst | 2 - ...-05-19-10-32-11.gh-issue-134152.INJC2j.rst | 2 - ...-05-19-15-05-24.gh-issue-134235.pz9PwV.rst | 2 - ...-05-19-20-59-06.gh-issue-134209.anhTcF.rst | 3 - ...5-05-20-15-13-43.gh-issue-86802.trF7TM.rst | 3 - ...-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst | 1 - ...5-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst | 2 - ...-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst | 1 - ...-05-22-14-12-53.gh-issue-134451.M1rD-j.rst | 1 - ...-05-22-18-14-13.gh-issue-134546.fjLVzK.rst | 1 - ...-05-23-23-43-39.gh-issue-134582.9POq3l.rst | 1 - ...5-05-24-03-10-36.gh-issue-80334.z21cMa.rst | 2 - ...-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst | 3 - ...-01-14-11-19-07.gh-issue-128840.M1doZW.rst | 2 - ...-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst | 2 - ...-05-08-15-06-01.gh-issue-133639.50-kbV.rst | 2 - ...-05-09-04-11-06.gh-issue-133682.-_lwo3.rst | 1 - ...-05-09-14-54-48.gh-issue-133744.LCquu0.rst | 3 - ...-05-19-14-57-46.gh-issue-134215.sbdDK6.rst | 1 - ...-05-08-19-07-26.gh-issue-133626.yFTKYK.rst | 2 - ...-05-13-13-25-27.gh-issue-133779.-YcTBz.rst | 6 - ...5-05-19-03-02-04.gh-issue-76023.vHOf6M.rst | 1 - ...-05-20-21-43-20.gh-issue-130727.-69t4D.rst | 2 - README.rst | 2 +- 78 files changed, 958 insertions(+), 276 deletions(-) create mode 100644 Misc/NEWS.d/3.14.0b2.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-04-16-09-38-48.gh-issue-117088.EFt_5c.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-04-30-10-22-08.gh-issue-131769.H0oy5x.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-05-21-19-46-28.gh-issue-134455.vdwlrq.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-05-17-14-41-21.gh-issue-134144.xVpZik.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-19-17-16-46.gh-issue-132542.7T_TY_.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-06-15-01-41.gh-issue-133516.RqWVf2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-07-23-26-53.gh-issue-133541.bHIC55.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-08-13-48-02.gh-issue-132762.tKbygC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-09-18-11-21.gh-issue-133778.pWEV3t.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-10-17-12-27.gh-issue-133703.bVM-re.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-11-13-40-42.gh-issue-133886.ryBAyo.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-15-11-38-16.gh-issue-133999.uBZ8uS.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-17-25-52.gh-issue-134100.5-FbLK.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-16-20-59-12.gh-issue-134119.w8expI.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-44-51.gh-issue-134158.ewLNLp.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-18-14-33-23.gh-issue-69605.ZMO49F.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-20-14-41-50.gh-issue-128066.qzzGfv.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-15-14-32.gh-issue-130397.aG6EON.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-21-18-02-56.gh-issue-127960.W3J_2X.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-22-14-48-19.gh-issue-134381.2BXhth.rst delete mode 100644 Misc/NEWS.d/next/Library/2017-12-30-18-21-00.bpo-28494.Dt_Wks.rst delete mode 100644 Misc/NEWS.d/next/Library/2022-07-24-20-56-32.gh-issue-69426.unccw7.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-13-21-41-34.gh-issue-86155.ppIGSC.rst delete mode 100644 Misc/NEWS.d/next/Library/2023-02-13-21-56-38.gh-issue-62824.CBZzX3.rst delete mode 100644 Misc/NEWS.d/next/Library/2024-10-28-06-54-22.gh-issue-125028.GEY8Ws.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-30-16-42-38.gh-issue-91555.ShVtwW.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-26-15-50-12.gh-issue-133009.etBuz5.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-29-11-48-46.gh-issue-132876.lyTQGZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-04-17-04-55.gh-issue-132493.huirKi.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-05-22-11-24.gh-issue-133439.LpmyFz.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-06-22-54-37.gh-issue-133551.rfy1tJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-07-19-16-41.gh-issue-133581.kERUCJ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-08-49-03.gh-issue-133701.KI8tGz.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-09-10-34.gh-issue-130328.s9h4By.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-15-50-00.gh-issue-77057.fV8SU-.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-18-29-25.gh-issue-133684.Y1DFSt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-19-05-24.gh-issue-133783.1voCnR.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-09-20-59-24.gh-issue-132641.3qTw44.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-10-12-06-55.gh-issue-133653.Gb2aG4.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-12-06-52-10.gh-issue-133925.elInBY.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-12-20-38-57.gh-issue-133960.Aee79f.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-13-18-21-59.gh-issue-71253.-3Sf_K.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-13-18-54-56.gh-issue-133970.6G-Oi6.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-15-14-27-01.gh-issue-134062.fRbJet.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-16-12-40-37.gh-issue-132124.T_5Odx.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-16-20-10-25.gh-issue-134098.YyTkKr.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-17-12-40-12.gh-issue-133889.Eh-zO4.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-17-13-46-20.gh-issue-134097.fgkjE1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-17-18-08-35.gh-issue-133890.onn9_X.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-17-20-23-57.gh-issue-133982.smS7au.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-18-12-48-39.gh-issue-62184.y11l10.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-18-13-23-29.gh-issue-134168.hgx3Xg.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-19-10-32-11.gh-issue-134152.INJC2j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-19-15-05-24.gh-issue-134235.pz9PwV.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-19-20-59-06.gh-issue-134209.anhTcF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-20-15-13-43.gh-issue-86802.trF7TM.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-20-19-16-30.gh-issue-134323.ZQZGvw.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-20-21-45-58.gh-issue-90871.Gkvtp6.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-22-13-10-32.gh-issue-114177.3TYUJ3.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-22-14-12-53.gh-issue-134451.M1rD-j.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-22-18-14-13.gh-issue-134546.fjLVzK.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-23-23-43-39.gh-issue-134582.9POq3l.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-24-03-10-36.gh-issue-80334.z21cMa.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-26-12-31-08.gh-issue-132710.ApU3TZ.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-01-14-11-19-07.gh-issue-128840.M1doZW.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-05-09-20-22-54.gh-issue-133767.kN2i3Q.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-05-08-15-06-01.gh-issue-133639.50-kbV.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-05-09-04-11-06.gh-issue-133682.-_lwo3.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-05-09-14-54-48.gh-issue-133744.LCquu0.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-05-19-14-57-46.gh-issue-134215.sbdDK6.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-05-08-19-07-26.gh-issue-133626.yFTKYK.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-05-13-13-25-27.gh-issue-133779.-YcTBz.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-05-19-03-02-04.gh-issue-76023.vHOf6M.rst delete mode 100644 Misc/NEWS.d/next/Windows/2025-05-20-21-43-20.gh-issue-130727.-69t4D.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 03ac5843b62066..aba26718b2e857 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,10 +21,10 @@ #define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.14.0b1+" +#define PY_VERSION "3.14.0b2" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 5f7e14a79d3356..804b15e9e2dcbd 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue May 6 18:33:44 2025 +# Autogenerated by Sphinx on Mon May 26 16:26:41 2025 # as part of the release process. topics = { @@ -435,9 +435,9 @@ async def func(param1, param2): 'atom-identifiers': r'''Identifiers (Names) ******************* -An identifier occurring as an atom is a name. See section Identifiers -and keywords for lexical definition and section Naming and binding for -documentation of naming and binding. +An identifier occurring as an atom is a name. See section Names +(identifiers and keywords) for lexical definition and section Naming +and binding for documentation of naming and binding. When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it @@ -1724,16 +1724,16 @@ class attributes; they are shared by instances. Instance attributes The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -3304,7 +3304,7 @@ def f() -> annotation: ... introspects and uses the annotations (such as "dataclasses" or "functools.singledispatch()"). -By default, annotations are lazily evaluated in a annotation scope. +By default, annotations are lazily evaluated in an annotation scope. This means that they are not evaluated when the code containing the annotation is evaluated. Instead, the interpreter saves information that can be used to evaluate the annotation later if requested. The @@ -3318,6 +3318,12 @@ def f() -> annotation: ... >>> f.__annotations__ {'param': 'annotation'} +This future statement will be deprecated and removed in a future +version of Python, but not before Python 3.13 reaches its end of life +(see **PEP 749**). When it is used, introspection tools like +"annotationlib.get_annotations()" and "typing.get_type_hints()" are +less likely to be able to resolve annotations at runtime. + -[ Footnotes ]- [1] The exception is propagated to the invocation stack unless there @@ -3832,7 +3838,7 @@ def double(x): You can also invoke "pdb" from the command line to debug other scripts. For example: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post- @@ -3856,6 +3862,23 @@ def double(x): Changed in version 3.7: Added the "-m" option. +-p, --pid + + Attach to the process with the specified PID. + + Added in version 3.14. + +To attach to a running Python process for remote debugging, use the +"-p" or "--pid" option with the target process’s PID: + + python -m pdb -p 1234 + +Note: + + Attaching to a process that is blocked in a system call or waiting + for I/O will only work once the next bytecode instruction is + executed or when the process receives a signal. + Typical usage to execute a statement under control of the debugger is: >>> import pdb @@ -5077,7 +5100,7 @@ class of the instance or a *non-virtual base class* thereof. The 'exprlists': r'''Expression lists **************** - starred_expression: ["*"] or_expr + starred_expression: "*" or_expr | expression flexible_expression: assignment_expression | starred_expression flexible_expression_list: flexible_expression ("," flexible_expression)* [","] starred_expression_list: starred_expression ("," starred_expression)* [","] @@ -5138,16 +5161,16 @@ class of the instance or a *non-virtual base class* thereof. The The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -5942,73 +5965,92 @@ class body. A "SyntaxError" is raised if a variable is used or to help avoid name clashes between “private” attributes of base and derived classes. See section Identifiers (Names). ''', - 'identifiers': r'''Identifiers and keywords -************************ + 'identifiers': r'''Names (identifiers and keywords) +******************************** -Identifiers (also referred to as *names*) are described by the -following lexical definitions. - -The syntax of identifiers in Python is based on the Unicode standard -annex UAX-31, with elaboration and changes as defined below; see also -**PEP 3131** for further details. +"NAME" tokens represent *identifiers*, *keywords*, and *soft +keywords*. Within the ASCII range (U+0001..U+007F), the valid characters for -identifiers include the uppercase and lowercase letters "A" through -"Z", the underscore "_" and, except for the first character, the -digits "0" through "9". Python 3.0 introduced additional characters -from outside the ASCII range (see **PEP 3131**). For these -characters, the classification uses the version of the Unicode -Character Database as included in the "unicodedata" module. +names include the uppercase and lowercase letters ("A-Z" and "a-z"), +the underscore "_" and, except for the first character, the digits "0" +through "9". + +Names must contain at least one character, but have no upper length +limit. Case is significant. + +Besides "A-Z", "a-z", "_" and "0-9", names can also use “letter-like” +and “number-like” characters from outside the ASCII range, as detailed +below. + +All identifiers are converted into the normalization form NFKC while +parsing; comparison of identifiers is based on NFKC. + +Formally, the first character of a normalized identifier must belong +to the set "id_start", which is the union of: -Identifiers are unlimited in length. Case is significant. +* Unicode category "" - uppercase letters (includes "A" to "Z") - identifier: xid_start xid_continue* - id_start: - id_continue: - xid_start: - xid_continue: +* Unicode category "" - lowercase letters (includes "a" to "z") -The Unicode category codes mentioned above stand for: +* Unicode category "" - titlecase letters -* *Lu* - uppercase letters +* Unicode category "" - modifier letters -* *Ll* - lowercase letters +* Unicode category "" - other letters -* *Lt* - titlecase letters +* Unicode category "" - letter numbers -* *Lm* - modifier letters +* {""_""} - the underscore -* *Lo* - other letters +* "" - an explicit set of characters in PropList.txt + to support backwards compatibility -* *Nl* - letter numbers +The remaining characters must belong to the set "id_continue", which +is the union of: -* *Mn* - nonspacing marks +* all characters in "id_start" -* *Mc* - spacing combining marks +* Unicode category "" - decimal numbers (includes "0" to "9") -* *Nd* - decimal numbers +* Unicode category "" - connector punctuations -* *Pc* - connector punctuations +* Unicode category "" - nonspacing marks -* *Other_ID_Start* - explicit list of characters in PropList.txt to - support backwards compatibility +* Unicode category "" - spacing combining marks -* *Other_ID_Continue* - likewise +* "" - another explicit set of characters in + PropList.txt to support backwards compatibility + +Unicode categories use the version of the Unicode Character Database +as included in the "unicodedata" module. + +These sets are based on the Unicode standard annex UAX-31. See also +**PEP 3131** for further details. + +Even more formally, names are described by the following lexical +definitions: -All identifiers are converted into the normal form NFKC while parsing; -comparison of identifiers is based on NFKC. + NAME: xid_start xid_continue* + id_start: | | | | | | "_" | + id_continue: id_start | | | | | + xid_start: + xid_continue: + identifier: -A non-normative HTML file listing all valid identifier characters for -Unicode 16.0.0 can be found at -https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +A non-normative listing of all valid identifier characters as defined +by Unicode is available in the DerivedCoreProperties.txt file in the +Unicode Character Database. Keywords ======== -The following identifiers are used as reserved words, or *keywords* of -the language, and cannot be used as ordinary identifiers. They must -be spelled exactly as written here: +The following names are used as reserved words, or *keywords* of the +language, and cannot be used as ordinary identifiers. They must be +spelled exactly as written here: False await else import pass None break except in raise @@ -6024,18 +6066,20 @@ class body. A "SyntaxError" is raised if a variable is used or Added in version 3.10. -Some identifiers are only reserved under specific contexts. These are -known as *soft keywords*. The identifiers "match", "case", "type" and -"_" can syntactically act as keywords in certain contexts, but this -distinction is done at the parser level, not when tokenizing. +Some names are only reserved under specific contexts. These are known +as *soft keywords*: + +* "match", "case", and "_", when used in the "match" statement. + +* "type", when used in the "type" statement. + +These syntactically act as keywords in their specific contexts, but +this distinction is done at the parser level, not when tokenizing. As soft keywords, their use in the grammar is possible while still preserving compatibility with existing code that uses these names as identifier names. -"match", "case", and "_" are used in the "match" statement. "type" is -used in the "type" statement. - Changed in version 3.12: "type" is now a soft keyword. @@ -6807,9 +6851,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -8845,9 +8889,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -9215,7 +9259,14 @@ class is used in a class pattern with positional arguments, each Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to - "len(s)". + "len(s)". For example: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' str.count(sub[, start[, end]]) @@ -9224,7 +9275,18 @@ class is used in a class pattern with positional arguments, each *end* are interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between - characters which is the length of the string plus one. + characters which is the length of the string plus one. For example: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 str.encode(encoding='utf-8', errors='strict') @@ -9389,7 +9451,7 @@ class is used in a class pattern with positional arguments, each str.isidentifier() Return "True" if the string is a valid identifier according to the - language definition, section Identifiers and keywords. + language definition, section Names (identifiers and keywords). "keyword.iskeyword()" can be used to test whether string "s" is a reserved identifier, such as "def" and "class". @@ -9421,8 +9483,8 @@ class is used in a class pattern with positional arguments, each str.isprintable() - Return true if all characters in the string are printable, false if - it contains at least one non-printable character. + Return "True" if all characters in the string are printable, + "False" if it contains at least one non-printable character. Here “printable” means the character is suitable for "repr()" to use in its output; “non-printable” means that "repr()" on built-in @@ -9669,6 +9731,18 @@ class is used in a class pattern with positional arguments, each >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is "None" and *maxsplit* is "0", only + leading runs of consecutive whitespace are considered. + + For example: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + str.splitlines(keepends=False) Return a list of the lines in the string, breaking at line @@ -11724,8 +11798,15 @@ class dict(iterable, **kwargs) the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal - to "{"one": 1, "two": 2, "three": 3}": + Providing keyword arguments as in the first example only works for + keys that are valid Python identifiers. Otherwise, any valid keys + can be used. + + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary + creation and equality, the following examples all return a + dictionary equal to "{"one": 1, "two": 2, "three": 3}": >>> a = dict(one=1, two=2, three=3) >>> b = {'one': 1, 'two': 2, 'three': 3} @@ -11740,6 +11821,29 @@ class dict(iterable, **kwargs) keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -11910,33 +12014,6 @@ class dict(iterable, **kwargs) Added in version 3.9. - Dictionaries compare equal if and only if they have the same "(key, - value)" pairs (regardless of ordering). Order comparisons (‘<’, - ‘<=’, ‘>=’, ‘>’) raise "TypeError". - - Dictionaries preserve insertion order. Note that updating a key - does not affect the order. Keys added after deletion are inserted - at the end. - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - Changed in version 3.7: Dictionary order is guaranteed to be - insertion order. This behavior was an implementation detail of - CPython from 3.6. - Dictionaries and dictionary views are reversible. >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} diff --git a/Misc/NEWS.d/3.14.0b2.rst b/Misc/NEWS.d/3.14.0b2.rst new file mode 100644 index 00000000000000..0ab8dd4d470a42 --- /dev/null +++ b/Misc/NEWS.d/3.14.0b2.rst @@ -0,0 +1,766 @@ +.. date: 2025-05-20-21-43-20 +.. gh-issue: 130727 +.. nonce: -69t4D +.. release date: 2025-05-26 +.. section: Windows + +Fix a race in internal calls into WMI that can result in an "invalid handle" +exception under high load. Patch by Chris Eibl. + +.. + +.. date: 2025-05-19-03-02-04 +.. gh-issue: 76023 +.. nonce: vHOf6M +.. section: Windows + +Make :func:`os.path.realpath` ignore Windows error 1005 when in non-strict +mode. + +.. + +.. date: 2025-05-13-13-25-27 +.. gh-issue: 133779 +.. nonce: -YcTBz +.. section: Windows + +Reverts the change to generate different :file:`pyconfig.h` files based on +compiler settings, as it was frequently causing extension builds to break. +In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always +be defined explicitly when compiling for the experimental free-threaded +runtime. The :func:`sysconfig.get_config_var` function can be used to +determine whether the current runtime was compiled with that flag or not. + +.. + +.. date: 2025-05-08-19-07-26 +.. gh-issue: 133626 +.. nonce: yFTKYK +.. section: Windows + +Ensures packages are not accidentally bundled into the traditional +installer. + +.. + +.. date: 2025-05-19-14-57-46 +.. gh-issue: 134215 +.. nonce: sbdDK6 +.. section: Tools/Demos + +:term:`REPL` import autocomplete only suggests private modules when +explicitly specified. + +.. + +.. date: 2025-05-09-14-54-48 +.. gh-issue: 133744 +.. nonce: LCquu0 +.. section: Tests + +Fix multiprocessing interrupt test. Add an event to synchronize the parent +process with the child process: wait until the child process starts +sleeping. Patch by Victor Stinner. + +.. + +.. date: 2025-05-09-04-11-06 +.. gh-issue: 133682 +.. nonce: -_lwo3 +.. section: Tests + +Fixed test case ``test.test_annotationlib.TestStringFormat.test_displays`` +which ensures proper handling of complex data structures (lists, sets, +dictionaries, and tuples) in string annotations. + +.. + +.. date: 2025-05-08-15-06-01 +.. gh-issue: 133639 +.. nonce: 50-kbV +.. section: Tests + +Fix ``TestPyReplAutoindent.test_auto_indent_default()`` doesn't run +``input_code``. + +.. + +.. date: 2025-05-09-20-22-54 +.. gh-issue: 133767 +.. nonce: kN2i3Q +.. section: Security + +Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error +handler. + +.. + +.. date: 2025-01-14-11-19-07 +.. gh-issue: 128840 +.. nonce: M1doZW +.. section: Security + +Short-circuit the processing of long IPv6 addresses early in +:mod:`ipaddress` to prevent excessive memory consumption and a minor +denial-of-service. + +.. + +.. date: 2025-05-26-12-31-08 +.. gh-issue: 132710 +.. nonce: ApU3TZ +.. section: Library + +If possible, ensure that :func:`uuid.getnode` returns the same result even +across different processes. Previously, the result was constant only within +the same process. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-24-03-10-36 +.. gh-issue: 80334 +.. nonce: z21cMa +.. section: Library + +:func:`multiprocessing.freeze_support` now checks for work on any "spawn" +start method platform rather than only on Windows. + +.. + +.. date: 2025-05-23-23-43-39 +.. gh-issue: 134582 +.. nonce: 9POq3l +.. section: Library + +Fix tokenize.untokenize() round-trip errors related to t-strings braces +escaping + +.. + +.. date: 2025-05-22-18-14-13 +.. gh-issue: 134546 +.. nonce: fjLVzK +.. section: Library + +Ensure :mod:`pdb` remote debugging script is readable by remote Python +process. + +.. + +.. date: 2025-05-22-14-12-53 +.. gh-issue: 134451 +.. nonce: M1rD-j +.. section: Library + +Converted ``asyncio.tools.CycleFoundException`` from dataclass to a regular +exception type. + +.. + +.. date: 2025-05-22-13-10-32 +.. gh-issue: 114177 +.. nonce: 3TYUJ3 +.. section: Library + +Fix :mod:`asyncio` to not close subprocess pipes which would otherwise error +out when the event loop is already closed. + +.. + +.. date: 2025-05-20-21-45-58 +.. gh-issue: 90871 +.. nonce: Gkvtp6 +.. section: Library + +Fixed an off by one error concerning the backlog parameter in +:meth:`~asyncio.loop.create_unix_server`. Contributed by Christian Harries. + +.. + +.. date: 2025-05-20-19-16-30 +.. gh-issue: 134323 +.. nonce: ZQZGvw +.. section: Library + +Fix the :meth:`threading.RLock.locked` method. + +.. + +.. date: 2025-05-20-15-13-43 +.. gh-issue: 86802 +.. nonce: trF7TM +.. section: Library + +Fixed asyncio memory leak in cancelled shield tasks. For shielded tasks +where the shield was cancelled, log potential exceptions through the +exception handler. Contributed by Christian Harries. + +.. + +.. date: 2025-05-19-20-59-06 +.. gh-issue: 134209 +.. nonce: anhTcF +.. section: Library + +:mod:`curses`: The :meth:`curses.window.instr` and +:meth:`curses.window.getstr` methods now allocate their internal buffer on +the heap instead of the stack; in addition, the max buffer size is increased +from 1023 to 2047. + +.. + +.. date: 2025-05-19-15-05-24 +.. gh-issue: 134235 +.. nonce: pz9PwV +.. section: Library + +Updated tab completion on REPL to include builtin modules. Contributed by +Tom Wang, Hunter Young + +.. + +.. date: 2025-05-19-10-32-11 +.. gh-issue: 134152 +.. nonce: INJC2j +.. section: Library + +Fixed :exc:`UnboundLocalError` that could occur during :mod:`email` header +parsing if an expected trailing delimiter is missing in some contexts. + +.. + +.. date: 2025-05-18-13-23-29 +.. gh-issue: 134168 +.. nonce: hgx3Xg +.. section: Library + +:mod:`http.server`: Fix IPv6 address binding and :option:`--directory +` handling when using HTTPS. + +.. + +.. date: 2025-05-18-12-48-39 +.. gh-issue: 62184 +.. nonce: y11l10 +.. section: Library + +Remove import of C implementation of :class:`io.FileIO` from Python +implementation which has its own implementation + +.. + +.. date: 2025-05-17-20-23-57 +.. gh-issue: 133982 +.. nonce: smS7au +.. section: Library + +Emit :exc:`RuntimeWarning` in the Python implementation of :mod:`io` when +the :term:`file-like object ` is not closed explicitly in the +presence of multiple I/O layers. + +.. + +.. date: 2025-05-17-18-08-35 +.. gh-issue: 133890 +.. nonce: onn9_X +.. section: Library + +The :mod:`tarfile` module now handles :exc:`UnicodeEncodeError` in the same +way as :exc:`OSError` when cannot extract a member. + +.. + +.. date: 2025-05-17-13-46-20 +.. gh-issue: 134097 +.. nonce: fgkjE1 +.. section: Library + +Fix interaction of the new :term:`REPL` and :option:`-X showrefcount <-X>` +command line option. + +.. + +.. date: 2025-05-17-12-40-12 +.. gh-issue: 133889 +.. nonce: Eh-zO4 +.. section: Library + +The generated directory listing page in +:class:`http.server.SimpleHTTPRequestHandler` now only shows the decoded +path component of the requested URL, and not the query and fragment. + +.. + +.. date: 2025-05-16-20-10-25 +.. gh-issue: 134098 +.. nonce: YyTkKr +.. section: Library + +Fix handling paths that end with a percent-encoded slash (``%2f`` or +``%2F``) in :class:`http.server.SimpleHTTPRequestHandler`. + +.. + +.. date: 2025-05-16-12-40-37 +.. gh-issue: 132124 +.. nonce: T_5Odx +.. section: Library + +On POSIX-compliant systems, :func:`!multiprocessing.util.get_temp_dir` now +ignores :envvar:`TMPDIR` (and similar environment variables) if the path +length of ``AF_UNIX`` socket files exceeds the platform-specific maximum +length when using the :ref:`forkserver +` start method. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-05-15-14-27-01 +.. gh-issue: 134062 +.. nonce: fRbJet +.. section: Library + +:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for +:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` objects. + +.. + +.. date: 2025-05-13-18-54-56 +.. gh-issue: 133970 +.. nonce: 6G-Oi6 +.. section: Library + +Make :class:`!string.templatelib.Template` and +:class:`!string.templatelib.Interpolation` generic. + +.. + +.. date: 2025-05-13-18-21-59 +.. gh-issue: 71253 +.. nonce: -3Sf_K +.. section: Library + +Raise :exc:`ValueError` in :func:`open` if *opener* returns a negative +file-descriptor in the Python implementation of :mod:`io` to match the C +implementation. + +.. + +.. date: 2025-05-12-20-38-57 +.. gh-issue: 133960 +.. nonce: Aee79f +.. section: Library + +Simplify and improve :func:`typing.evaluate_forward_ref`. It now no longer +raises errors on certain invalid types. In several situations, it is now +able to evaluate forward references that were previously unsupported. + +.. + +.. date: 2025-05-12-06-52-10 +.. gh-issue: 133925 +.. nonce: elInBY +.. section: Library + +Make the private class ``typing._UnionGenericAlias`` hashable. + +.. + +.. date: 2025-05-10-12-06-55 +.. gh-issue: 133653 +.. nonce: Gb2aG4 +.. section: Library + +Fix :class:`argparse.ArgumentParser` with the *formatter_class* argument. +Fix TypeError when *formatter_class* is a custom subclass of +:class:`!HelpFormatter`. Fix TypeError when *formatter_class* is not a +subclass of :class:`!HelpFormatter` and non-standard *prefix_char* is used. +Fix support of colorizing when *formatter_class* is not a subclass of +:class:`!HelpFormatter`. + +.. + +.. date: 2025-05-09-20-59-24 +.. gh-issue: 132641 +.. nonce: 3qTw44 +.. section: Library + +Fixed a race in :func:`functools.lru_cache` under free-threading. + +.. + +.. date: 2025-05-09-19-05-24 +.. gh-issue: 133783 +.. nonce: 1voCnR +.. section: Library + +Fix bug with applying :func:`copy.replace` to :mod:`ast` objects. Attributes +that default to ``None`` were incorrectly treated as required for manually +created AST nodes. + +.. + +.. date: 2025-05-09-18-29-25 +.. gh-issue: 133684 +.. nonce: Y1DFSt +.. section: Library + +Fix bug where :func:`annotationlib.get_annotations` would return the wrong +result for certain classes that are part of a class hierarchy where ``from +__future__ import annotations`` is used. + +.. + +.. date: 2025-05-09-15-50-00 +.. gh-issue: 77057 +.. nonce: fV8SU- +.. section: Library + +Fix handling of invalid markup declarations in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-05-09-09-10-34 +.. gh-issue: 130328 +.. nonce: s9h4By +.. section: Library + +Speedup pasting in ``PyREPL`` on Windows in a legacy console. Patch by Chris +Eibl. + +.. + +.. date: 2025-05-09-08-49-03 +.. gh-issue: 133701 +.. nonce: KI8tGz +.. section: Library + +Fix bug where :class:`typing.TypedDict` classes defined under ``from +__future__ import annotations`` and inheriting from another ``TypedDict`` +had an incorrect ``__annotations__`` attribute. + +.. + +.. date: 2025-05-07-19-16-41 +.. gh-issue: 133581 +.. nonce: kERUCJ +.. section: Library + +Improve unparsing of t-strings in :func:`ast.unparse` and ``from __future__ +import annotations``. Empty t-strings now round-trip correctly and +formatting in interpolations is preserved. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-06-22-54-37 +.. gh-issue: 133551 +.. nonce: rfy1tJ +.. section: Library + +Support t-strings (:pep:`750`) in :mod:`annotationlib`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2025-05-05-22-11-24 +.. gh-issue: 133439 +.. nonce: LpmyFz +.. section: Library + +Fix dot commands with trailing spaces are mistaken for multi-line SQL +statements in the sqlite3 command-line interface. + +.. + +.. date: 2025-05-04-17-04-55 +.. gh-issue: 132493 +.. nonce: huirKi +.. section: Library + +Avoid accessing ``__annotations__`` unnecessarily in +:func:`inspect.signature`. + +.. + +.. date: 2025-04-29-11-48-46 +.. gh-issue: 132876 +.. nonce: lyTQGZ +.. section: Library + +``ldexp()`` on Windows doesn't round subnormal results before Windows 11, +but should. Python's :func:`math.ldexp` wrapper now does round them, so +results may change slightly, in rare cases of very small results, on Windows +versions before 11. + +.. + +.. date: 2025-04-26-15-50-12 +.. gh-issue: 133009 +.. nonce: etBuz5 +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +` when the element is concurrently mutated. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-03-30-16-42-38 +.. gh-issue: 91555 +.. nonce: ShVtwW +.. section: Library + +Ignore log messages generated during handling of log messages, to avoid +deadlock or infinite recursion. + +.. + +.. date: 2024-10-28-06-54-22 +.. gh-issue: 125028 +.. nonce: GEY8Ws +.. section: Library + +:data:`functools.Placeholder` cannot be passed to :func:`functools.partial` +as a keyword argument. + +.. + +.. date: 2023-02-13-21-56-38 +.. gh-issue: 62824 +.. nonce: CBZzX3 +.. section: Library + +Fix aliases for ``iso8859_8`` encoding. Patch by Dave Goncalves. + +.. + +.. date: 2023-02-13-21-41-34 +.. gh-issue: 86155 +.. nonce: ppIGSC +.. section: Library + +:meth:`html.parser.HTMLParser.close` no longer loses data when the +```. * Multiple slashes and whitespaces between the last attribute and closing `>` are now ignored in both start and end tags. E.g. ``. * Multiple `=` between attribute name and value are no longer collapsed. E.g. `` produces attribute "foo" with value "=bar". * Whitespaces between the `=` separator and attribute name or value are no longer ignored. E.g. `` produces two attributes "foo" and "=bar", both with value None; `` produces two attributes: "foo" with value "" and "bar" with value None. --------- (cherry picked from commit 0243f97cbadec8d985e63b1daec5d1cbc850cae3) Co-authored-by: Serhiy Storchaka Co-authored-by: Ezio Melotti --- Lib/html/parser.py | 143 ++++++++-------- Lib/test/test_htmlparser.py | 155 +++++++++++------- ...-06-25-14-13-39.gh-issue-135661.idjQ0B.rst | 25 +++ 3 files changed, 194 insertions(+), 129 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index ba416e7fa6e3fe..cc15de07b5bae6 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -31,15 +31,43 @@ piclose = re.compile('>') commentclose = re.compile(r'--\s*>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + (= # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:= # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -56,8 +84,6 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') # Character reference processing logic specific to attribute values @@ -141,7 +167,8 @@ def get_starttag_text(self): def set_cdata_mode(self, elem): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal @@ -166,7 +193,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -310,7 +337,7 @@ def parse_html_declaration(self, i): return self.parse_bogus_comment(i) # Internal -- parse bogus comment, return length or -1 if not terminated - # see http://www.w3.org/TR/html5/tokenization.html#bogus-comment-state + # see https://html.spec.whatwg.org/multipage/parsing.html#bogus-comment-state def parse_bogus_comment(self, i, report=1): rawdata = self.rawdata assert rawdata[i:i+2] in ('": - return j + 1 - if next == "/": - if rawdata.startswith("/>", j): - return j + 2 - if rawdata.startswith("/", j): - # buffer boundary - return -1 - # else bogus input - if j > i: - return j - else: - return i + 1 - if next == "": - # end of input - return -1 - if next in ("abcdefghijklmnopqrstuvwxyz=/" - "ABCDEFGHIJKLMNOPQRSTUVWXYZ"): - # end of input in or before attribute value, or we have the - # '/' from a '/>' ending - return -1 - if j > i: - return j - else: - return i + 1 - raise AssertionError("we should not get here!") + match = locatetagend.match(rawdata, i+1) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 + return j # Internal -- parse endtag, return end or -1 if incomplete def parse_endtag(self, i): + # See the HTML5 specs section "13.2.5.7 End tag open state" + # https://html.spec.whatwg.org/multipage/parsing.html#end-tag-open-state rawdata = self.rawdata assert rawdata[i:i+2] == " - if not match: + if rawdata.find('>', i+2) < 0: # fast check return -1 - gtpos = match.end() - match = endtagfind.match(rawdata, i) # - if not match: - if self.cdata_elem is not None: - self.handle_data(rawdata[i:gtpos]) - return gtpos - # find the name: w3.org/TR/html5/tokenization.html#tag-name-state - namematch = tagfind_tolerant.match(rawdata, i+2) - if not namematch: - # w3.org/TR/html5/tokenization.html#end-tag-open-state - if rawdata[i:i+3] == '': - return i+3 - else: - return self.parse_bogus_comment(i) - tagname = namematch.group(1).lower() - # consume and ignore other stuff between the name and the > - # Note: this is not 100% correct, since we might have things like - # , but looking for > after the name should cover - # most of the cases and is much simpler - gtpos = rawdata.find('>', namematch.end()) - self.handle_endtag(tagname) - return gtpos+1 + if not endtagopen.match(rawdata, i): # ': # is ignored + # "missing-end-tag-name" parser error + return i+3 + else: + return self.parse_bogus_comment(i) - elem = match.group(1).lower() # script or style - if self.cdata_elem is not None: - if elem != self.cdata_elem: - self.handle_data(rawdata[i:gtpos]) - return gtpos + match = locatetagend.match(rawdata, i+2) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 - self.handle_endtag(elem) + # find the name: "13.2.5.8 Tag name state" + # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state + match = tagfind_tolerant.match(rawdata, i+2) + assert match + tag = match.group(1).lower() + self.handle_endtag(tag) self.clear_cdata_mode() - return gtpos + return j # Overridable -- finish processing of start+end tag: def handle_startendtag(self, tag, attrs): diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 65a4bee72b9775..d0d2c54217ccaf 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -81,6 +81,13 @@ def handle_entityref(self, data): self.fail('This should never be called with convert_charrefs=True') +# The normal event collector normalizes the events in get_events, +# so we override it to return the original list of events. +class EventCollectorNoNormalize(EventCollector): + def get_events(self): + return self.events + + class TestCaseBase(unittest.TestCase): def get_collector(self): @@ -265,8 +272,7 @@ def test_get_starttag_text(self): ("starttag", "foo:bar", [("one", "1"), ("two", "2")]), ("starttag_text", s)]) - def test_cdata_content(self): - contents = [ + @support.subTests('content', [ ' ¬-an-entity-ref;', "", '

', @@ -279,54 +285,83 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) - - def test_cdata_with_closing_tags(self): + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + 'foo = ""', + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + 'a::before { content: "< /style>"; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + 'a::before { content: ""; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """ ¬-an-entity-ref;

''""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("" @@ -443,7 +478,7 @@ def test_starttag_junk_chars(self): self._run_check("", [('comment', '$')]) self._run_check("", [('endtag', 'a')]) + self._run_check("", [('comment', ' a')]) self._run_check("", [('starttag', 'a", [('endtag', 'a', [('endtag', 'a')]) + self._run_check('', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("", [('decl', 'DOCTYPE foo $ ')]) @@ -525,15 +564,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = 'This confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -560,7 +595,7 @@ def test_correct_detection_of_start_tags(self): html = '

The rain' expected = [ - ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]), + ('starttag', 'div', [('style', ''), (',', None), ('foo', None), ('=', None), ('"bar"', None)]), ('starttag', 'b', []), ('data', 'The '), ('starttag', 'a', [('href', 'some_url')]), @@ -749,9 +784,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) - self._run_check("""""", output) + self._run_check("", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""""", @@ -760,6 +801,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -831,7 +876,7 @@ def test_malformed_attributes(self): ('data', 'test - bad2'), ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), ('data', 'test - bad3'), ('endtag', 'a'), - ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), + ('starttag', 'a', [('href', None), ('=', None), ("test' style", 'color:red;bad4')]), ('data', 'test - bad4'), ('endtag', 'a') ] self._run_check(html, expected) diff --git a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst new file mode 100644 index 00000000000000..b6f9e104e44047 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst @@ -0,0 +1,25 @@ +Fix parsing start and end tags in :class:`html.parser.HTMLParser` +according to the HTML5 standard. + +* Whitespaces no longer accepted between ```` does not end the script section. + +* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized + as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. + +* Null character (U+0000) no longer ends the tag name. + +* Attributes and slashes after the tag name in end tags are now ignored, + instead of terminating after the first ``>`` in quoted attribute value. + E.g. ````. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ````. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ```` produces attribute "foo" with value "=bar". + +* Whitespaces between the ``=`` separator and attribute name or value are no + longer ignored. E.g. ```` produces two attributes "foo" and + "=bar", both with value None; ```` produces two attributes: + "foo" with value "" and "bar" with value None. From 2449f7773446b45cc3bae2cc502e8fedf896df8d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 3 Jul 2025 23:10:27 +0200 Subject: [PATCH 549/713] [3.14] Clarify some details regarding `sys.monitoring` (GH-133981) (GH-136259) Clarify some details regarding `sys.monitoring` (GH-133981) (cherry picked from commit b4991056f4f44acb50aea3838463e749c24c7420) Co-authored-by: Brett Cannon --- Doc/library/sys.monitoring.rst | 61 ++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0674074b8c0df6..f62a4011e4144b 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -137,7 +137,8 @@ The following events are supported: .. monitoring-event:: PY_UNWIND - Exit from a Python function during exception unwinding. + Exit from a Python function during exception unwinding. This includes exceptions raised directly within the + function and that are allowed to continue to propagate. .. monitoring-event:: PY_YIELD @@ -171,7 +172,7 @@ events, use the expression ``PY_RETURN | PY_START``. if get_events(DEBUGGER_ID) == NO_EVENTS: ... -Events are divided into three groups: + Setting this event deactivates all events. .. _monitoring-event-local: @@ -243,20 +244,23 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike +:monitoring-event:`RAISE`. -Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` -event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable -when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for -performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. +Note that the :monitoring-event:`STOP_ITERATION` event and the +:monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are +equivalent, and are treated as interchangeable when generating events. +Implementations will favor :monitoring-event:`STOP_ITERATION` for performance +reasons, but may generate a :monitoring-event:`RAISE` event with a +:exc:`StopIteration`. Turning events on and off ------------------------- In order to monitor an event, it must be turned on and a corresponding callback -must be registered. -Events can be turned on or off by setting the events either globally or -for a particular code object. +must be registered. Events can be turned on or off by setting the events either +globally and/or for a particular code object. An event will trigger only once, +even if it is turned on both globally and locally. Setting events globally @@ -292,10 +296,6 @@ in Python (see :ref:`c-api-monitoring`). Activates all the local events for *code* which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not in use. -Local events add to global events, but do not mask them. -In other words, all global events will trigger for a code object, -regardless of the local events. - Disabling events '''''''''''''''' @@ -325,8 +325,6 @@ except for a few breakpoints. Registering callback functions ------------------------------ -To register a callable for events call - .. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None Registers the callable *func* for the *event* with the given *tool_id* @@ -335,12 +333,16 @@ To register a callable for events call it is unregistered and returned. Otherwise :func:`register_callback` returns ``None``. - Functions can be unregistered by calling ``sys.monitoring.register_callback(tool_id, event, None)``. Callback functions can be registered and unregistered at any time. +Callbacks are called only once regardless if the event is turned on both +globally and locally. As such, if an event could be turned on for both global +and local events by your code then the callback needs to be written to handle +either trigger. + Registering or unregistering a callback function will generate a :func:`sys.audit` event. @@ -353,37 +355,46 @@ Callback function arguments that there are no arguments to the call. When an active event occurs, the registered callback function is called. +Callback functions returning an object other than :data:`DISABLE` will have no effect. Different events will provide the callback function with different arguments, as follows: * :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object * :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, retval: object) -> object -* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN` + (*arg0* can be :data:`MISSING` specifically):: - func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, callable: object, arg0: object) -> object + *code* represents the code object where the call is being made, while + *callable* is the object that is about to be called (and thus + triggered the event). If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. + For instance methods, *callable* will be the function object as found on the + class with *arg0* set to the instance (i.e. the ``self`` argument to the + method). + * :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, exception: BaseException) -> object * :monitoring-event:`LINE`:: - func(code: CodeType, line_number: int) -> DISABLE | Any + func(code: CodeType, line_number: int) -> object * :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: - func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, destination_offset: int) -> object Note that the *destination_offset* is where the code will next execute. * :monitoring-event:`INSTRUCTION`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object From fa7e76e4ddbe92da1f1d98a24cc15d8306ef4e2e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 4 Jul 2025 09:26:03 +0200 Subject: [PATCH 550/713] [3.14] gh-102555: Fix comment parsing in HTMLParser according to the HTML5 standard (GH-135664) (GH-136271) * "--!>" now ends the comment. * "-- >" no longer ends the comment. * Support abnormally ended empty comments "<-->" and "<--->". --------- (cherry picked from commit 8ac7613dc8b8f82253d7c0e2b6ef6ed703a0a1ee) Co-author: Kerim Kabirov Co-authored-by: Serhiy Storchaka Co-authored-by: Ezio Melotti --- Lib/html/parser.py | 18 ++++++++++- Lib/test/test_htmlparser.py | 32 +++++++++++++++++-- ...-06-18-13-28-08.gh-issue-102555.nADrzJ.rst | 3 ++ 3 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index cc15de07b5bae6..9b4f09599134bd 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -29,7 +29,8 @@ starttagopen = re.compile('<[a-zA-Z]') endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: # 1) if you change tagfind/attrfind remember to update locatetagend too; # 2) if you change tagfind/attrfind and/or locatetagend the parser will @@ -336,6 +337,21 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith('" '' '' + '' '' + # abrupt-closing-of-empty-comment + '' + '' '' '' - '') + '' + '' + '' + '' + '' + '' + '' + # nested-comment + ' -->' + '' + '' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' '), + ('comment', '`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. From f2f3edb8d7daba96a73741a269ceb61575957633 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 4 Jul 2025 16:36:12 +0200 Subject: [PATCH 551/713] [3.14] gh-135773: have pyvenv.cfg without home key anchor a venv and deduce home (GH-135831) (#136287) gh-135773: have pyvenv.cfg without home key anchor a venv and deduce home (GH-135831) This is still formally undefined behaviour, but we may as well keep the *same* undefined behaviour as previous versions. PEP 796 proposes a cleaner and more consistent replacement for 3.15+ (cherry picked from commit 93263d43141a81d369adfcddf325f9a54cb5766d) Co-authored-by: Richard Levasseur --- Lib/test/test_getpath.py | 21 +++++++++++++++++++++ Modules/getpath.py | 11 ++++------- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f86df9d0d03485..83f09f3495547a 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -354,6 +354,27 @@ def test_venv_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_posix_without_home_key(self): + ns = MockPosixNamespace( + argv0="/venv/bin/python3", + PREFIX="/usr", + ENV_PATH="/usr/bin", + ) + # Setup the bare minimum venv + ns.add_known_xfile("/usr/bin/python3") + ns.add_known_xfile("/venv/bin/python3") + ns.add_known_link("/venv/bin/python3", "/usr/bin/python3") + ns.add_known_file("/venv/pyvenv.cfg", [ + # home = key intentionally omitted + ]) + expected = dict( + executable="/venv/bin/python3", + prefix="/venv", + base_prefix="/usr", + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_changed_name_posix(self): "Test a venv layout on *nix." ns = MockPosixNamespace( diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345afbda..b89d7427e3febd 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -364,10 +364,9 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] - # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the - # presence of a pyvenv.cfg file without a 'home' key to signify the - # existence of a virtual environment — we quietly ignore them. - # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! + # Search for the 'home' key in pyvenv.cfg. If a home key isn't found, + # then it means a venv is active and home is based on the venv's + # executable (if its a symlink, home is where the symlink points). for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -412,10 +411,8 @@ def search_up(prefix, *landmarks, test=isfile): if isfile(candidate): base_executable = candidate break + # home key found; stop iterating over lines break - else: - # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. - venv_prefix = None # ****************************************************************************** From 1a2898a383ac5b57a7e0eba34ccab2055d79f1aa Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 4 Jul 2025 16:46:57 +0200 Subject: [PATCH 552/713] [3.14] Docs: Move "or" outside monospace syntax in `tarfile.rst` (GH-136263) (GH-136290) Docs: Move "or" outside monospace syntax in `tarfile.rst` (GH-136263) (cherry picked from commit ade19880943509945da193202ca89e0b2b6fbd75) Co-authored-by: Rafael Fontenelle --- Doc/library/tarfile.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f1298514bd5b0e..8d10db8f2c2921 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -63,8 +63,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | mode | action | +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | + | ``'r'`` or | Open for reading with transparent | + | ``'r:*'`` | compression (recommended). | +------------------+---------------------------------------------+ | ``'r:'`` | Open for reading exclusively without | | | compression. | @@ -98,10 +98,11 @@ Some facts and figures: | | Raise a :exc:`FileExistsError` exception | | | if it already exists. | +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | + | ``'a'`` or | Open for appending with no compression. The | + | ``'a:'`` | file is created if it does not exist. | +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | + | ``'w'`` or | Open for uncompressed writing. | + | ``'w:'`` | | +------------------+---------------------------------------------+ | ``'w:gz'`` | Open for gzip compressed writing. | +------------------+---------------------------------------------+ From 6d21cc54fffbe56df3372f65227160bf27807158 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 4 Jul 2025 18:18:32 +0200 Subject: [PATCH 553/713] [3.14] gh-136288: Fix error message in `_testcapi/vectorcall.c` (GH-136258) (GH-136294) gh-136288: Fix error message in `_testcapi/vectorcall.c` (GH-136258) Use the %N format specifier instead of %s and `PyType_GetName`. (cherry picked from commit d1d5dce14f90d777608e4403d09079421ff55944) Co-authored-by: William S Fulton --- Modules/_testcapi/vectorcall.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 03aaacb328e0b6..f89dcb6c4cf03c 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -179,14 +179,14 @@ _testcapi_VectorCallClass_set_vectorcall_impl(PyObject *self, if (!PyObject_TypeCheck(self, type)) { return PyErr_Format( PyExc_TypeError, - "expected %s instance", - PyType_GetName(type)); + "expected %N instance", + type); } if (!type->tp_vectorcall_offset) { return PyErr_Format( PyExc_TypeError, - "type %s has no vectorcall offset", - PyType_GetName(type)); + "type %N has no vectorcall offset", + type); } *(vectorcallfunc*)((char*)self + type->tp_vectorcall_offset) = ( VectorCallClass_vectorcall); From c62c523e034ddef262e90c16c6e2fef69ff7d469 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 5 Jul 2025 08:39:48 +0200 Subject: [PATCH 554/713] [3.14] gh-136297: Test all `pickle` protocols in `test_zoneinfo_property.py` (GH-136298) (#136311) gh-136297: Test all `pickle` protocols in `test_zoneinfo_property.py` (GH-136298) (cherry picked from commit 5de7e3f9739b01ad180fffb242ac57cea930e74d) Co-authored-by: sobolevn --- .../test_zoneinfo/test_zoneinfo_property.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index feaa77f3e7f0b9..294c7e9b27a857 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -146,20 +146,24 @@ def setUp(self): @add_key_examples def test_pickle_unpickle_cache(self, key): zi = self.klass(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIs(zi, zi_rt) + self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIsNot(zi, zi_rt) - self.assertEqual(str(zi), str(zi_rt)) + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) @add_key_examples From 53584d307add2c6f32e4bf9a34a3ffa310a1dcd0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 5 Jul 2025 15:35:24 +0200 Subject: [PATCH 555/713] [3.14] gh-101100: Fix references in `http.cookiejar` docs (GH-136238) (GH-136318) gh-101100: Fix references in `http.cookiejar` docs (GH-136238) (cherry picked from commit f0c7344a8fbfe67c05981cce67562e7facfae73d) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/library/http.cookiejar.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 23ddecf873876d..251aea891c3f98 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -570,7 +570,7 @@ Netscape protocol strictness switches: Don't allow setting cookies whose path doesn't path-match request URI. -:attr:`strict_ns_domain` is a collection of flags. Its value is constructed by +:attr:`~DefaultCookiePolicy.strict_ns_domain` is a collection of flags. Its value is constructed by or-ing together (for example, ``DomainStrictNoDots|DomainStrictNonDomain`` means both flags are set). From 2b2cf81a6c84d24b9d669716d0fdccb923801941 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 5 Jul 2025 20:24:19 +0200 Subject: [PATCH 556/713] [3.14] gh-136315: Fix skipped multithreading test in test_zstd (GH-136320) (#136322) gh-136315: Fix skipped multithreading test in test_zstd (GH-136320) Fix skipped test in test_zstd (cherry picked from commit 5dac137b9f75c5c1d5096101bcd33d565d0526e4) Co-authored-by: Emma Smith --- Lib/test/test_zstd.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 7bd0ffd9391529..e83caaf4c07b13 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -63,15 +63,18 @@ TRAINED_DICT = None -SUPPORT_MULTITHREADING = False +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False C_INT_MIN = -(2**31) C_INT_MAX = (2**31) - 1 def setUpModule(): - global SUPPORT_MULTITHREADING - SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) # uncompressed size 130KB, more than a zstd block. # with a frame epilogue, 4 bytes checksum. global DAT_130K_D From d86ca7b610fb8cf70796f40c51ef4f0332970a08 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 6 Jul 2025 10:13:13 +0200 Subject: [PATCH 557/713] [3.14] gh-136285: Improve `pickle` protocol testing in `test_interpreters` (GH-136286) (#136333) gh-136285: Improve `pickle` protocol testing in `test_interpreters` (GH-136286) (cherry picked from commit 06e347b84648f3f8e144e8f70671d610da082b77) Co-authored-by: sobolevn --- Lib/concurrent/interpreters/__init__.py | 8 ++------ Lib/concurrent/interpreters/_queues.py | 8 ++------ Lib/test/support/channels.py | 8 ++------ Lib/test/test_interpreters/test_api.py | 8 +++++--- Lib/test/test_interpreters/test_channels.py | 16 ++++++++++------ Lib/test/test_interpreters/test_queues.py | 8 +++++--- ...025-07-05-09-45-04.gh-issue-136286.N67Amr.rst | 2 ++ 7 files changed, 28 insertions(+), 30 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst diff --git a/Lib/concurrent/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py index 0fd661249a276c..aa46a2b37a48d5 100644 --- a/Lib/concurrent/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -146,12 +146,8 @@ def __del__(self): self._decref() # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (self._id,)) def _decref(self): if not self._ownsref: diff --git a/Lib/concurrent/interpreters/_queues.py b/Lib/concurrent/interpreters/_queues.py index d6a3197d9e0e26..ee830973f2a36e 100644 --- a/Lib/concurrent/interpreters/_queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -130,12 +130,8 @@ def __hash__(self): return hash(self._id) # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (self._id,)) def _set_unbound(self, op, items=None): assert not hasattr(self, '_unbound') diff --git a/Lib/test/support/channels.py b/Lib/test/support/channels.py index dfa86ba24dc134..5352f7d4da3af0 100644 --- a/Lib/test/support/channels.py +++ b/Lib/test/support/channels.py @@ -105,12 +105,8 @@ def __eq__(self, other): return other._id == self._id # for pickling: - def __getnewargs__(self): - return (int(self._id),) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (int(self._id),)) @property def id(self): diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 0ee4582b5d1568..a34b20beaca7a3 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -412,9 +412,11 @@ def test_equality(self): def test_pickle(self): interp = interpreters.create() - data = pickle.dumps(interp) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, interp) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(interp, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) class TestInterpreterIsRunning(TestBase): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 109ddf344539ad..52827357078b85 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -121,9 +121,11 @@ def test_equality(self): def test_pickle(self): ch, _ = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendChannelAttrs(TestBase): @@ -152,9 +154,11 @@ def test_equality(self): def test_pickle(self): _, ch = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendRecv(TestBase): diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 6b106fa21aab7d..815d38d8bd7a4a 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -189,9 +189,11 @@ def test_equality(self): def test_pickle(self): queue = queues.create() - data = pickle.dumps(queue) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, queue) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(queue, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) class TestQueueOps(TestBase): diff --git a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst new file mode 100644 index 00000000000000..0a0d66ac0b8abf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst @@ -0,0 +1,2 @@ +Fix pickling failures for protocols 0 and 1 for many objects realted to +subinterpreters. From 3ea7276a8e9edcf5cc5409448467023a34d4f083 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 6 Jul 2025 15:23:55 +0200 Subject: [PATCH 558/713] [3.14] gh-129824: fix data races in subinterpreters under TSAN (GH-135794) (#136266) gh-129824: fix data races in subinterpreters under TSAN (GH-135794) This fixes the data races in typeobject.c in subinterpreters under free-threading. The type flags and slots are only modified in the main interpreter as all static types are first initialised in main interpreter. (cherry picked from commit b582d751b4968d4534fdb7894e50713676789b2f) Co-authored-by: Kumar Aditya --- Objects/typeobject.c | 61 ++++++++++++++++++++++++++++++++------------ 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 455d32434c4688..a348ea1d531c93 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -54,7 +54,6 @@ class object "PyObject *" "&PyBaseObject_Type" PyUnicode_CheckExact(name) && \ (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE) -#define NEXT_GLOBAL_VERSION_TAG _PyRuntime.types.next_version_tag #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag @@ -157,8 +156,8 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - int64_t interp_count = - _PyRuntime.types.managed_static.types[full_index].interp_count; + int64_t interp_count = _Py_atomic_load_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count); assert((interp_count == 0) == (_PyRuntime.types.managed_static.types[full_index].type == NULL)); *p_interp_count = interp_count; @@ -235,7 +234,7 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; assert((initial == 1) == - (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) == 0)); (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); @@ -284,7 +283,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, : &(interp->types.for_extensions.initialized[index]); assert(state != NULL); - assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) > 0); assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); @@ -294,7 +293,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); if (final) { - assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + assert(!_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count)); _PyRuntime.types.managed_static.types[full_index].type = NULL; managed_static_type_index_clear(self); @@ -1258,6 +1257,19 @@ _PyType_GetVersionForCurrentState(PyTypeObject *tp) #error "_Py_ATTR_CACHE_UNUSED must be bigger than max" #endif +static inline unsigned int +next_global_version_tag(void) +{ + unsigned int old; + do { + old = _Py_atomic_load_uint_relaxed(&_PyRuntime.types.next_version_tag); + if (old >= _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + return 0; + } + } while (!_Py_atomic_compare_exchange_uint(&_PyRuntime.types.next_version_tag, &old, old + 1)); + return old + 1; +} + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -1288,11 +1300,12 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ - if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + unsigned int next_version_tag = next_global_version_tag(); + if (next_version_tag == 0) { /* We have run out of version numbers */ return 0; } - set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); + set_version_unlocked(type, next_version_tag); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -8704,7 +8717,11 @@ type_ready_set_new(PyTypeObject *type, int initial) && base == &PyBaseObject_Type && !(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + if (initial) { + type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + } else { + assert(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION); + } } if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { @@ -8718,13 +8735,17 @@ type_ready_set_new(PyTypeObject *type, int initial) } } else { - // tp_new is NULL: inherit tp_new from base - type->tp_new = base->tp_new; + if (initial) { + // tp_new is NULL: inherit tp_new from base + type->tp_new = base->tp_new; + } } } else { // Py_TPFLAGS_DISALLOW_INSTANTIATION sets tp_new to NULL - type->tp_new = NULL; + if (initial) { + type->tp_new = NULL; + } } return 0; } @@ -8857,7 +8878,12 @@ type_ready(PyTypeObject *type, int initial) } /* All done -- set the ready flag */ - type_add_flags(type, Py_TPFLAGS_READY); + if (initial) { + type_add_flags(type, Py_TPFLAGS_READY); + } else { + assert(type->tp_flags & Py_TPFLAGS_READY); + } + stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -8906,15 +8932,16 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(initial); + if (initial) { + assert((self->tp_flags & Py_TPFLAGS_READY) == 0); type_add_flags(self, _Py_TPFLAGS_STATIC_BUILTIN); type_add_flags(self, Py_TPFLAGS_IMMUTABLETYPE); - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); if (self->tp_version_tag == 0) { - _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + unsigned int next_version_tag = next_global_version_tag(); + assert(next_version_tag != 0); + _PyType_SetVersion(self, next_version_tag); } } else { From 5b65df7925d0d6c9c837661a5a0348dda4a88ba6 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 6 Jul 2025 17:43:16 +0200 Subject: [PATCH 559/713] [3.14] gh-109700: fix memory error handling in `PyDict_SetDefault` (GH-136338) (#136340) gh-109700: fix memory error handling in `PyDict_SetDefault` (GH-136338) (cherry picked from commit d22e073d2b49313bbf42d40cbe74afa2b69385df) Co-authored-by: Kumar Aditya --- .../2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst | 1 + Objects/dictobject.c | 1 + 2 files changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst new file mode 100644 index 00000000000000..a37f4a51050947 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst @@ -0,0 +1 @@ +Fix memory error handling in :c:func:`PyDict_SetDefault`. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 72901377cbb3da..b4940315a6c28d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4411,6 +4411,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu if (result) { *result = NULL; } + return -1; } STORE_USED(mp, mp->ma_used + 1); From bd97cb822922c4577238e376d1ab0b28aa475e9d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 6 Jul 2025 20:32:03 +0200 Subject: [PATCH 560/713] [3.14] gh-109070: Document that get_context in multiprocessing have side effect (GH-136341) (GH-136343) gh-109070: Document that get_context in multiprocessing have side effect (GH-136341) Document that get_context in multiprocessing have side effect (cherry picked from commit 77a8bd29da99e7d4fa8e7f07c4063977c2bb14d3) Co-authored-by: AN Long --- Doc/library/multiprocessing.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6f37930ea80391..b41953d21442c9 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -1118,7 +1118,9 @@ Miscellaneous Return a context object which has the same attributes as the :mod:`multiprocessing` module. - If *method* is ``None`` then the default context is returned. + If *method* is ``None`` then the default context is returned. Note that if + the global start method has not been set, this will set it to the + default method. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1129,10 +1131,10 @@ Miscellaneous Return the name of start method used for starting processes. - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. + If the global start method has not been set and *allow_none* is + ``False``, then the start method is set to the default and the name + is returned. If the start method has not been set and *allow_none* is + ``True`` then ``None`` is returned. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. From 3d01565da4dfca00c2db28f99e710e40e405a5d1 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Sun, 6 Jul 2025 17:10:57 -0700 Subject: [PATCH 561/713] [3.14] gh-136316: Make typing.evaluate_forward_ref better at evaluating nested forwardrefs (GH-136319) (#136346) (cherry picked from commit 9312702d2e12c2f58f02bfa02877d0ec790d06bd) --- Lib/test/test_typing.py | 6 ++++++ Lib/test/typinganndata/fwdref_module.py | 6 ++++++ Lib/typing.py | 12 +++++++++--- .../2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst | 2 ++ 4 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 Lib/test/typinganndata/fwdref_module.py create mode 100644 Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 983fb39f4dbdcd..cd552f9887d8a0 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -7328,6 +7328,12 @@ def test_partial_evaluation(self): list[EqualToForwardRef('A')], ) + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) + class CollectionsAbcTests(BaseTestCase): diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 00000000000000..7347a7a42455c2 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/typing.py b/Lib/typing.py index 87fd53802ca36d..09a574d783c8b4 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -430,7 +430,7 @@ def __repr__(self): def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), - format=None, owner=None): + format=None, owner=None, parent_fwdref=None): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -451,7 +451,7 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( - _make_forward_ref(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -936,7 +936,12 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -def _make_forward_ref(code, **kwargs): +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ @@ -1001,6 +1006,7 @@ def evaluate_forward_ref( recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, format=format, owner=owner, + parent_fwdref=forward_ref, ) diff --git a/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst b/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst new file mode 100644 index 00000000000000..dd5cecdf3a1209 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst @@ -0,0 +1,2 @@ +Improve support for evaluating nested forward references in +:func:`typing.evaluate_forward_ref`. From 22fad16ebc707706d18d067ed12cbbe0c8181a03 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 09:40:13 +0200 Subject: [PATCH 562/713] [3.14] gh-109700: fix interpreter finalization while handling memory error (GH-136342) (#136352) gh-109700: fix interpreter finalization while handling memory error (GH-136342) (cherry picked from commit 0c3e3da19570424649c33c0c2c29dc12541935e7) Co-authored-by: Kumar Aditya --- Python/pylifecycle.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c4c1d9fd9e1380..c07862edf97d0b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1702,8 +1702,10 @@ finalize_modules(PyThreadState *tstate) #endif // Stop watching __builtin__ modifications - PyDict_Unwatch(0, interp->builtins); - + if (PyDict_Unwatch(0, interp->builtins) < 0) { + // might happen if interp is cleared before watching the __builtin__ + PyErr_Clear(); + } PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -2377,15 +2379,13 @@ new_interpreter(PyThreadState **tstate_p, error: *tstate_p = NULL; if (tstate != NULL) { - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + Py_EndInterpreter(tstate); + } else { + PyInterpreterState_Delete(interp); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyInterpreterState_Delete(interp); - return status; } From aa1800b75513a67b47ed8075970158975329e321 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:08:41 +0200 Subject: [PATCH 563/713] [3.14] gh-127502: Remove XML vulnerability table (GH-135294) (#136359) gh-127502: Remove XML vulnerability table (GH-135294) * Remove the table * Replace warnings with notes Latest releases of Python 3.9-3.15 include expat 2.7.1 which is not vulnerable. expat 2.6.0 was released in February 2024. (cherry picked from commit cb99d992774b67761441e122965ed056bac09241) Co-authored-by: Victor Stinner --- Doc/library/pyexpat.rst | 7 ++- Doc/library/security_warnings.rst | 2 +- Doc/library/xml.dom.minidom.rst | 7 ++- Doc/library/xml.dom.pulldom.rst | 7 ++- Doc/library/xml.etree.elementtree.rst | 7 ++- Doc/library/xml.rst | 76 +++++---------------------- Doc/library/xml.sax.rst | 7 ++- Doc/library/xmlrpc.client.rst | 4 +- Doc/library/xmlrpc.server.rst | 4 +- 9 files changed, 32 insertions(+), 89 deletions(-) diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2d57cff10a9278..5506ac828e5abe 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -16,11 +16,10 @@ references to these attributes should be marked using the :member: role. -.. warning:: +.. note:: - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. index:: single: Expat diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst index a573c98f73eb0a..70c359cc1c0fc3 100644 --- a/Doc/library/security_warnings.rst +++ b/Doc/library/security_warnings.rst @@ -28,7 +28,7 @@ The following modules have specific security considerations: ` * :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race conditions ` -* :mod:`xml`: :ref:`XML vulnerabilities ` +* :mod:`xml`: :ref:`XML security ` * :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume exhaustion ` diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 00a18751207e7a..9ffedf7366a7b8 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -19,11 +19,10 @@ not already proficient with the DOM should consider using the :mod:`xml.etree.ElementTree` module for their XML processing instead. -.. warning:: +.. note:: - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. DOM applications typically start by parsing some XML into a DOM. With diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst index fd96765cbe3c96..8bceeecd46393e 100644 --- a/Doc/library/xml.dom.pulldom.rst +++ b/Doc/library/xml.dom.pulldom.rst @@ -19,11 +19,10 @@ responsible for explicitly pulling events from the stream, looping over those events until either processing is finished or an error condition occurs. -.. warning:: +.. note:: - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 1daf6628013bf0..00075ac2a23e6b 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -20,11 +20,10 @@ for parsing and creating XML data. The :mod:`!xml.etree.cElementTree` module is deprecated. -.. warning:: +.. note:: - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. Tutorial -------- diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index d495995398959d..28465219a1ac18 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -15,12 +15,10 @@ XML Processing Modules Python's interfaces for processing XML are grouped in the ``xml`` package. -.. warning:: +.. note:: - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defusedxml-package` sections. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -47,46 +45,22 @@ The XML handling submodules are: * :mod:`xml.parsers.expat`: the Expat parser binding +.. _xml-security: .. _xml-vulnerabilities: -XML vulnerabilities -------------------- +XML security +------------ -The XML processing modules are not secure against maliciously constructed data. An attacker can abuse XML features to carry out denial of service attacks, access local files, generate network connections to other machines, or circumvent firewalls. -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. - -========================= ================== ================== ================== ================== ================== -kind sax etree minidom pulldom xmlrpc -========================= ================== ================== ================== ================== ================== -billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) -`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -large tokens **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) **Vulnerable** (6) -========================= ================== ================== ================== ================== ================== - -1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and - "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. -2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. -3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. -5. Since Python 3.7.1, external general entities are no longer processed by - default. -6. Expat 2.6.0 and newer is not vulnerable to denial of service - through quadratic runtime caused by parsing large tokens. - Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`!pyexpat.EXPAT_VERSION`. +Expat versions lower that 2.6.0 may be vulnerable to "billion laughs", +"quadratic blowup" and "large tokens". Python may be vulnerable if it uses such +older versions of Expat as a system-provided library. +Check :const:`!pyexpat.EXPAT_VERSION`. + +:mod:`xmlrpc` is **vulnerable** to the "decompression bomb" attack. billion laughs / exponential entity expansion @@ -103,16 +77,6 @@ quadratic blowup entity expansion efficient as the exponential case but it avoids triggering parser countermeasures that forbid deeply nested entities. -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -`DTD`_ retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - decompression bomb Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries that can parse compressed XML streams such as gzipped HTTP streams or @@ -126,21 +90,5 @@ large tokens be used to cause denial of service in the application parsing XML. The issue is known as :cve:`2023-52425`. -The documentation for :pypi:`defusedxml` on PyPI has further information about -all known attack vectors with examples and references. - -.. _defusedxml-package: - -The :mod:`!defusedxml` Package ------------------------------- - -:pypi:`defusedxml` is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - - .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst index c60e9e505f7544..5fa92645a440ce 100644 --- a/Doc/library/xml.sax.rst +++ b/Doc/library/xml.sax.rst @@ -18,11 +18,10 @@ SAX exceptions and the convenience functions which will be most used by users of the SAX API. -.. warning:: +.. note:: - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. versionchanged:: 3.7.1 diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index 654154cb43d6e5..547cb50be78a78 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -24,8 +24,8 @@ between conformable Python objects and XML on the wire. .. warning:: The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. versionchanged:: 3.5 diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst index 06169c7eca8b0c..2a8f6f8d5fc0de 100644 --- a/Doc/library/xmlrpc.server.rst +++ b/Doc/library/xmlrpc.server.rst @@ -20,8 +20,8 @@ servers written in Python. Servers can either be free standing, using .. warning:: The :mod:`xmlrpc.server` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + constructed data. If you need to parse untrusted or unauthenticated data, + see :ref:`xml-security`. .. include:: ../includes/wasm-notavail.rst From 8f06a9c9ceff5174ac091e956dee3b67dea133b9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:18:27 +0200 Subject: [PATCH 564/713] [3.14] gh-94503: Update logging cookbook with an example of uniformly handling newlines in output. (GH-136217) (GH-136357) (cherry picked from commit d05423a90ce0ee9ad5207dce3dd06ab2397f3d6e) --- Doc/howto/logging-cookbook.rst | 62 ++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7d64a02358adb3..ae2697fbce30ad 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4078,6 +4078,68 @@ lines. With this approach, you get better output: WARNING:demo: 1/0 WARNING:demo:ZeroDivisionError: division by zero +How to uniformly handle newlines in logging output +-------------------------------------------------- + +Usually, messages that are logged (say to console or file) consist of a single +line of text. However, sometimes there is a need to handle messages with +multiple lines - whether because a logging format string contains newlines, or +logged data contains newlines. If you want to handle such messages uniformly, so +that each line in the logged message appears uniformly formatted as if it was +logged separately, you can do this using a handler mixin, as in the following +snippet: + +.. code-block:: python + + # Assume this is in a module mymixins.py + import copy + + class MultilineMixin: + def emit(self, record): + s = record.getMessage() + if '\n' not in s: + super().emit(record) + else: + lines = s.splitlines() + rec = copy.copy(record) + rec.args = None + for line in lines: + rec.msg = line + super().emit(rec) + +You can use the mixin as in the following script: + +.. code-block:: python + + import logging + + from mymixins import MultilineMixin + + logger = logging.getLogger(__name__) + + class StreamHandler(MultilineMixin, logging.StreamHandler): + pass + + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-9s %(message)s', + handlers = [StreamHandler()]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +The script, when run, prints something like: + +.. code-block:: text + + 2025-07-02 13:54:47,234 DEBUG Single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me once ... + 2025-07-02 13:54:47,234 DEBUG Another single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me ... + 2025-07-02 13:54:47,234 DEBUG can't get fooled again + .. patterns-to-avoid: From 46a7981f45ee81512e5f06d3afaa0f0ecbe7f1ec Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:31:18 +0200 Subject: [PATCH 565/713] [3.14] GH-135106: Restrict trashcan to GC'ed objects (GH-135682) (#135876) Co-authored-by: Mark Shannon --- ...-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst | 2 + Objects/object.c | 76 ++++++------------- 2 files changed, 26 insertions(+), 52 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst new file mode 100644 index 00000000000000..b6e953a7719be1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst @@ -0,0 +1,2 @@ +Restrict the trashcan mechanism to GC'ed objects and untrack them while in +the trashcan to prevent the GC and trashcan mechanisms conflicting. diff --git a/Objects/object.c b/Objects/object.c index 7140520a1bb3e8..8dc3ab9643cbb2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2923,57 +2923,28 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#ifndef Py_GIL_DISABLED -/* We need to store a pointer in the refcount field of - * an object. It is important that we never store 0 (NULL). -* It is also important to not make the object appear immortal, -* or it might be untracked by the cycle GC. */ -static uintptr_t -pointer_to_safe_refcount(void *ptr) -{ - uintptr_t full = (uintptr_t)ptr; - assert((full & 3) == 0); -#if SIZEOF_VOID_P > 4 - uint32_t refcnt = (uint32_t)full; - if (refcnt >= (uint32_t)_Py_IMMORTAL_MINIMUM_REFCNT) { - full = full - ((uintptr_t)_Py_IMMORTAL_MINIMUM_REFCNT) + 1; - } - return full + 2; -#else - // Make the top two bits 0, so it appears mortal. - return (full >> 2) + 1; -#endif -} - -static void * -safe_refcount_to_pointer(uintptr_t refcnt) -{ -#if SIZEOF_VOID_P > 4 - if (refcnt & 1) { - refcnt += _Py_IMMORTAL_MINIMUM_REFCNT - 1; - } - return (void *)(refcnt - 2); -#else - return (void *)((refcnt -1) << 2); -#endif -} -#endif - /* Add op to the gcstate->trash_delete_later list. Called when the current - * call-stack depth gets large. op must be a currently untracked gc'ed - * object, with refcount 0. Py_DECREF must already have been called on it. + * call-stack depth gets large. op must be a gc'ed object, with refcount 0. + * Py_DECREF must already have been called on it. */ void _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, Py_REFCNT(op) == 0); + PyTypeObject *tp = Py_TYPE(op); + assert(tp->tp_flags & Py_TPFLAGS_HAVE_GC); + int tracked = 0; + if (tp->tp_is_gc == NULL || tp->tp_is_gc(op)) { + tracked = _PyObject_GC_IS_TRACKED(op); + if (tracked) { + _PyObject_GC_UNTRACK(op); + } + } + uintptr_t tagged_ptr = ((uintptr_t)tstate->delete_later) | tracked; #ifdef Py_GIL_DISABLED - op->ob_tid = (uintptr_t)tstate->delete_later; + op->ob_tid = tagged_ptr; #else - /* Store the delete_later pointer in the refcnt field. */ - uintptr_t refcnt = pointer_to_safe_refcount(tstate->delete_later); - *((uintptr_t*)op) = refcnt; - assert(!_Py_IsImmortal(op)); + _Py_AS_GC(op)->_gc_next = tagged_ptr; #endif tstate->delete_later = op; } @@ -2988,17 +2959,17 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate) destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_GIL_DISABLED - tstate->delete_later = (PyObject*) op->ob_tid; + uintptr_t tagged_ptr = op->ob_tid; op->ob_tid = 0; _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else - /* Get the delete_later pointer from the refcnt field. - * See _PyTrash_thread_deposit_object(). */ - uintptr_t refcnt = *((uintptr_t*)op); - tstate->delete_later = safe_refcount_to_pointer(refcnt); - op->ob_refcnt = 0; + uintptr_t tagged_ptr = _Py_AS_GC(op)->_gc_next; + _Py_AS_GC(op)->_gc_next = 0; #endif - + tstate->delete_later = (PyObject *)(tagged_ptr & ~1); + if (tagged_ptr & 1) { + _PyObject_GC_TRACK(op); + } /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but * Py_DECREF was already called on this object, and in @@ -3072,10 +3043,11 @@ void _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); + unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC; destructor dealloc = type->tp_dealloc; PyThreadState *tstate = _PyThreadState_GET(); intptr_t margin = _Py_RecursionLimit_GetMargin(tstate); - if (margin < 2) { + if (margin < 2 && gc_flag) { _PyTrash_thread_deposit_object(tstate, (PyObject *)op); return; } @@ -3121,7 +3093,7 @@ _Py_Dealloc(PyObject *op) Py_XDECREF(old_exc); Py_DECREF(type); #endif - if (tstate->delete_later && margin >= 4) { + if (tstate->delete_later && margin >= 4 && gc_flag) { _PyTrash_thread_destroy_chain(tstate); } } From 41d1683e0548f14bee84b21ccf1fd837b340da04 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:33:55 +0200 Subject: [PATCH 566/713] [3.14] GH-133932: Tagged ints are heap safe (GH-134244) (#134494) Co-authored-by: Mark Shannon --- Include/internal/pycore_stackref.h | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 4b5caca5dbd126..21306b95f9c2e4 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -239,6 +239,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else #define Py_INT_TAG 3 +#define Py_TAG_REFCNT 1 static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) @@ -264,7 +265,7 @@ PyStackRef_UntagInt(_PyStackRef i) #ifdef Py_GIL_DISABLED -#define Py_TAG_DEFERRED (1) +#define Py_TAG_DEFERRED Py_TAG_REFCNT #define Py_TAG_PTR ((uintptr_t)0) #define Py_TAG_BITS ((uintptr_t)1) @@ -442,14 +443,13 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ -#define Py_TAG_BITS 1 -#define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" #endif #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits)) -#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_BITS))) +#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT))) #define PyStackRef_NULL_BITS Py_TAG_REFCNT static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; @@ -529,7 +529,7 @@ PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); #if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_BITS; + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; #else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; #endif @@ -548,12 +548,6 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) return ref; } -// Check if a stackref is exactly the same as another stackref, including the -// the deferred bit. This can only be used safely if you know that the deferred -// bits of `a` and `b` match. -#define PyStackRef_IsExactly(a, b) \ - (assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits) - static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { @@ -606,7 +600,7 @@ PyStackRef_DUP(_PyStackRef ref) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) == 0 || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); } static inline _PyStackRef @@ -681,7 +675,7 @@ PyStackRef_XCLOSE(_PyStackRef ref) // Note: this is a macro because MSVC (Windows) has trouble inlining it. -#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_BITS)) == ((b).bits & (~Py_TAG_BITS))) +#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) From 57d6db529b29413f1da014a6c74b6c794f89b381 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:43:44 +0200 Subject: [PATCH 567/713] [3.14] gh-136289: Fix test_sqlite3 on platforms with strict UTF-8 filesystem (GH-136326) (GH-136350) (cherry picked from commit 85b817da94cf911a964d42e81a57e1de9ab71ef9) Co-authored-by: Serhiy Storchaka --- Lib/test/test_sqlite3/test_dbapi.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index c3aa3bf2d7bd9f..0ef05ff1929b24 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -31,8 +31,7 @@ import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_apple, is_emscripten, is_wasi + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) from test.support import gc_collect from test.support import threading_helper, import_helper @@ -649,14 +648,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + def get_undecodable_path(self): + path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -696,14 +702,10 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) From b2a2f032dcd82c676ab315374994227ca3a398d3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:50:02 +0200 Subject: [PATCH 568/713] [3.14] gh-130662: Accept leading zeros in precision/width for Fraction's formatting (GH-130663) (#136361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sergey B Kirpichev Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/fractions.py | 4 ++-- Lib/test/test_fractions.py | 8 ++------ .../2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst | 3 +++ 3 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index ef296fd6a0f632..aeae91ee4220e1 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -168,9 +168,9 @@ def _round_to_figures(n, d, figures): # A '0' that's *not* followed by another digit is parsed as a minimum width # rather than a zeropad flag. (?P0(?=[0-9]))? - (?P0|[1-9][0-9]*)? + (?P[0-9]+)? (?P[,_])? - (?:\.(?P0|[1-9][0-9]*))? + (?:\.(?P[0-9]+))? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 50020cd8ac5a84..1875a2f529c957 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1515,6 +1515,8 @@ def test_format_f_presentation_type(self): (F(51, 1000), '.1f', '0.1'), (F(149, 1000), '.1f', '0.1'), (F(151, 1000), '.1f', '0.2'), + (F(22, 7), '.02f', '3.14'), # issue gh-130662 + (F(22, 7), '005.02f', '03.14'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1613,12 +1615,6 @@ def test_invalid_formats(self): '=010%', '>00.2f', '>00f', - # Too many zeros - minimum width should not have leading zeros - '006f', - # Leading zeros in precision - '.010f', - '.02f', - '.000f', # Missing precision '.e', '.f', diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst new file mode 100644 index 00000000000000..e07200f9a3fbbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst @@ -0,0 +1,3 @@ +Accept leading zeros in precision and width fields for +:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, +3), '.016f')``. From f87578f8317342e3795d3f2af39689388d3e6888 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 11:55:35 +0200 Subject: [PATCH 569/713] [3.14] gh-130662: Accept leading zeros in precision/width for Decimal's formatting (GH-132549) (#136362) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sergey B Kirpichev Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/_pydecimal.py | 4 ++-- Lib/test/test_decimal.py | 2 ++ .../Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst | 3 +++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 46fa9ffcb1e056..781b38ec26ba33 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -6120,9 +6120,9 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? +(?P\d+)? (?P[,_])? -(?:\.(?P0|(?!0)\d+))? +(?:\.(?P\d+))? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 9e298401dc3dcc..ca2b0b18fb0a6d 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -982,6 +982,7 @@ def test_formatting(self): ('.0f', '0e-2', '0'), ('.0f', '3.14159265', '3'), ('.1f', '3.14159265', '3.1'), + ('.01f', '3.14159265', '3.1'), # leading zero in precision ('.4f', '3.14159265', '3.1416'), ('.6f', '3.14159265', '3.141593'), ('.7f', '3.14159265', '3.1415926'), # round-half-even! @@ -1067,6 +1068,7 @@ def test_formatting(self): ('8,', '123456', ' 123,456'), ('08,', '123456', '0,123,456'), # special case: extra 0 needed ('+08,', '123456', '+123,456'), # but not if there's a sign + ('008,', '123456', '0,123,456'), # leading zero in width (' 08,', '123456', ' 123,456'), ('08,', '-123456', '-123,456'), ('+09,', '123456', '+0,123,456'), diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst new file mode 100644 index 00000000000000..d97d937376a9d0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst @@ -0,0 +1,3 @@ ++Accept leading zeros in precision and width fields for ++:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), +'.016f')``. From f02be2dfe81c9a35eb1f03e082d5ca517a0bb790 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:48:04 +0200 Subject: [PATCH 570/713] [3.14] gh-101100: Fix sphinx warnings in `whatsnew/3.9` (GH-136163) (#136372) Co-authored-by: sobolevn --- Doc/library/random.rst | 5 +++++ Doc/tools/.nitignore | 1 - Doc/whatsnew/3.9.rst | 10 +++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Doc/library/random.rst b/Doc/library/random.rst index ef0cfb0e76cef6..b1120b3a4d8eb4 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -447,6 +447,11 @@ Alternative Generator Override this method in subclasses to customise the :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + .. method:: Random.randbytes(n) + + Override this method in subclasses to customise the + :meth:`~random.randbytes` behaviour of :class:`!Random` instances. + .. class:: SystemRandom([seed]) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index e3bcb968128d96..88f22db56f9684 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -73,6 +73,5 @@ Doc/whatsnew/3.5.rst Doc/whatsnew/3.6.rst Doc/whatsnew/3.7.rst Doc/whatsnew/3.8.rst -Doc/whatsnew/3.9.rst Doc/whatsnew/3.10.rst Doc/whatsnew/3.11.rst diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 896e8f4a489649..7fd9e6ac66e6c8 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -423,8 +423,8 @@ digests. It skips MD5 on platforms that block MD5 digest. fcntl ----- -Added constants :const:`~fcntl.F_OFD_GETLK`, :const:`~fcntl.F_OFD_SETLK` -and :const:`~fcntl.F_OFD_SETLKW`. +Added constants :const:`!fcntl.F_OFD_GETLK`, :const:`!fcntl.F_OFD_SETLK` +and :const:`!fcntl.F_OFD_SETLKW`. (Contributed by Donghee Na in :issue:`38602`.) ftplib @@ -644,7 +644,7 @@ attribute. random ------ -Added a new :attr:`random.Random.randbytes` method: generate random bytes. +Added a new :meth:`random.Random.randbytes` method: generate random bytes. (Contributed by Victor Stinner in :issue:`40286`.) signal @@ -776,7 +776,7 @@ Optimizations :pep:`590` vectorcall protocol. (Contributed by Donghee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) -* Optimized :func:`~set.difference_update` for the case when the other set +* Optimized :meth:`!set.difference_update` for the case when the other set is much larger than the base set. (Suggested by Evgeny Kapun with code contributed by Michele Orrù in :issue:`8425`.) @@ -1139,7 +1139,7 @@ Changes in the Python API (Contributed by Christian Heimes in :issue:`36384`). * :func:`codecs.lookup` now normalizes the encoding name the same way as - :func:`encodings.normalize_encoding`, except that :func:`codecs.lookup` also + :func:`!encodings.normalize_encoding`, except that :func:`codecs.lookup` also converts the name to lower case. For example, ``"latex+latin1"`` encoding name is now normalized to ``"latex_latin1"``. (Contributed by Jordon Xu in :issue:`37751`.) From 5df4f353ad935cc65e3f056671f6d11a544ed15c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:55:20 +0200 Subject: [PATCH 571/713] [3.14] gh-135755: Document __future__.* and CO_* as proper Sphinx objects (GH-135980) (GH-136370) * Turn the __future__ table to list-table. This'll make it easier to add entries that need longer markup * Semantic markup for __future__ feature descriptions. * Document CO_* C macros. (cherry picked from commit 2468aafe984fdf923811ef0c6969e3d6c1b92a82) Co-authored-by: Petr Viktorin --- Doc/c-api/code.rst | 65 +++++++++++++++++++++++++++++++ Doc/c-api/veryhigh.rst | 8 ++-- Doc/library/__future__.rst | 78 ++++++++++++++++++++++---------------- 3 files changed, 115 insertions(+), 36 deletions(-) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 42594f063b0709..717b0da8f87c7f 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -211,6 +211,71 @@ bound into a function. .. versionadded:: 3.12 +.. _c_codeobject_flags: + +Code Object Flags +----------------- + +Code objects contain a bit-field of flags, which can be retrieved as the +:attr:`~codeobject.co_flags` Python attribute (for example using +:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to +:c:func:`PyUnstable_Code_New` and similar functions. + +Flags whose names start with ``CO_FUTURE_`` correspond to features normally +selectable by :ref:`future statements `. These flags can be used in +:c:member:`PyCompilerFlags.cf_flags`. +Note that many ``CO_FUTURE_`` flags are mandatory in current versions of +Python, and setting them has no effect. + +The following flags are available. +For their meaning, see the linked documentation of their Python equivalents. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Flag + * Meaning + * * .. c:macro:: CO_OPTIMIZED + * :py:data:`inspect.CO_OPTIMIZED` + * * .. c:macro:: CO_NEWLOCALS + * :py:data:`inspect.CO_NEWLOCALS` + * * .. c:macro:: CO_VARARGS + * :py:data:`inspect.CO_VARARGS` + * * .. c:macro:: CO_VARKEYWORDS + * :py:data:`inspect.CO_VARKEYWORDS` + * * .. c:macro:: CO_NESTED + * :py:data:`inspect.CO_NESTED` + * * .. c:macro:: CO_GENERATOR + * :py:data:`inspect.CO_GENERATOR` + * * .. c:macro:: CO_COROUTINE + * :py:data:`inspect.CO_COROUTINE` + * * .. c:macro:: CO_ITERABLE_COROUTINE + * :py:data:`inspect.CO_ITERABLE_COROUTINE` + * * .. c:macro:: CO_ASYNC_GENERATOR + * :py:data:`inspect.CO_ASYNC_GENERATOR` + * * .. c:macro:: CO_HAS_DOCSTRING + * :py:data:`inspect.CO_HAS_DOCSTRING` + * * .. c:macro:: CO_METHOD + * :py:data:`inspect.CO_METHOD` + + * * .. c:macro:: CO_FUTURE_DIVISION + * no effect (:py:data:`__future__.division`) + * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT + * no effect (:py:data:`__future__.absolute_import`) + * * .. c:macro:: CO_FUTURE_WITH_STATEMENT + * no effect (:py:data:`__future__.with_statement`) + * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION + * no effect (:py:data:`__future__.print_function`) + * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS + * no effect (:py:data:`__future__.unicode_literals`) + * * .. c:macro:: CO_FUTURE_GENERATOR_STOP + * no effect (:py:data:`__future__.generator_stop`) + * * .. c:macro:: CO_FUTURE_ANNOTATIONS + * :py:data:`__future__.annotations` + + Extra information ----------------- diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 1ef4181d52eb10..fb07fec7effce8 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -361,7 +361,7 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. - .. c:var:: int CO_FUTURE_DIVISION - - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such + as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally + selectable using :ref:`future statements `. + See :ref:`c_codeobject_flags` for a complete list. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 4f3b663006fb28..5d916b30112d3c 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -37,38 +37,52 @@ No feature description will ever be deleted from :mod:`__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evaluation of | -| | | | annotations using descriptors* | -+------------------+-------------+--------------+---------------------------------------------+ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * feature + * optional in + * mandatory in + * effect + * * .. data:: nested_scopes + * 2.1.0b1 + * 2.2 + * :pep:`227`: *Statically Nested Scopes* + * * .. data:: generators + * 2.2.0a1 + * 2.3 + * :pep:`255`: *Simple Generators* + * * .. data:: division + * 2.2.0a2 + * 3.0 + * :pep:`238`: *Changing the Division Operator* + * * .. data:: absolute_import + * 2.5.0a1 + * 3.0 + * :pep:`328`: *Imports: Multi-Line and Absolute/Relative* + * * .. data:: with_statement + * 2.5.0a1 + * 2.6 + * :pep:`343`: *The “with” Statement* + * * .. data:: print_function + * 2.6.0a2 + * 3.0 + * :pep:`3105`: *Make print a function* + * * .. data:: unicode_literals + * 2.6.0a2 + * 3.0 + * :pep:`3112`: *Bytes literals in Python 3000* + * * .. data:: generator_stop + * 3.5.0b1 + * 3.7 + * :pep:`479`: *StopIteration handling inside generators* + * * .. data:: annotations + * 3.7.0b1 + * Never [1]_ + * :pep:`563`: *Postponed evaluation of annotations*, + :pep:`649`: *Deferred evaluation of annotations using descriptors* .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. From 236498c8457c2c7f3e03ab423231ac41bc5bbd8d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:07:23 +0200 Subject: [PATCH 572/713] [3.14] gh-131591: Document Py_REMOTE_DEBUG (GH-135929) (GH-136374) gh-131591: Document Py_REMOTE_DEBUG (GH-135929) (cherry picked from commit 9aac5a3d44d7191001660aed3a383b062ee4b29b) Co-authored-by: Petr Viktorin --- Doc/using/configure.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index a26d48d7d0a345..e5fe3c72b1b26e 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -686,6 +686,13 @@ also be used to improve performance. not compiled. This includes both the functionality to schedule code to be executed and the functionality to receive code to be executed. + .. c:macro:: Py_REMOTE_DEBUG + + This macro is defined by default, unless Python is configured with + :option:`--without-remote-debug`. + + Note that even if the macro is defined, remote debugging may not be + available (for example, on an incompatible platform). .. versionadded:: 3.14 From a7dd48acbaa7eff6972fb3b96cbce6325eecb0d9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 13:27:54 +0200 Subject: [PATCH 573/713] [3.14] gh-86682: Add versionadded for sys._getframemodulename (GH-136325) (#136375) gh-86682: Add versionadded for sys._getframemodulename (GH-136325) add versionadded for sys._getframemodulename (cherry picked from commit 11f074b243756bca0db5a7d35dd84f00879de616) Co-authored-by: Anthony Sottile --- Doc/library/sys.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 1626a89a0731bf..05bc7cfb9dc089 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -953,6 +953,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only This function should be used for internal and specialized purposes only. It is not guaranteed to exist in all implementations of Python. + .. versionadded:: 3.12 + .. function:: getobjects(limit[, type]) From d9c36a7aa6a570de8dea3a30c8eb82b13fcc80a2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:57:37 +0200 Subject: [PATCH 574/713] [3.14] gh-124486: Fix test_whichdb_ndbm in test_dbm on NetBSD (GH-136335) (GH-136378) On NetBSD, ndbm.open() does not fail for empty file. (cherry picked from commit b7aa2a4b4df697db6ea45a555eeb3fefa5ca5bd4) Co-authored-by: Serhiy Storchaka --- Lib/test/test_dbm.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index a10922a403ec38..014c190285c402 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -213,7 +213,8 @@ def test_whichdb(self): @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - with open(_fname + '.db', 'wb'): pass + with open(_fname + '.db', 'wb') as f: + f.write(b'spam') _bytes_fname = os.fsencode(_fname) fnames = [_fname, os_helper.FakePath(_fname), _bytes_fname, os_helper.FakePath(_bytes_fname)] From 442b781cfc012558106f00f4a026ca3f22587f77 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:24:24 +0200 Subject: [PATCH 575/713] [3.14] gh-87790: support thousands separators for formatting fractional part of Fraction (GH-132204) (#136364) Co-authored-by: Sergey B Kirpichev --- Lib/fractions.py | 10 +++++++++- Lib/test/test_fractions.py | 13 +++++++++++++ .../2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst | 2 ++ 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst diff --git a/Lib/fractions.py b/Lib/fractions.py index aeae91ee4220e1..a497ee19935345 100644 --- a/Lib/fractions.py +++ b/Lib/fractions.py @@ -170,7 +170,11 @@ def _round_to_figures(n, d, figures): (?P0(?=[0-9]))? (?P[0-9]+)? (?P[,_])? - (?:\.(?P[0-9]+))? + (?:\. + (?=[,_0-9]) # lookahead for digit or separator + (?P[0-9]+)? + (?P[,_])? + )? (?P[eEfFgG%]) """, re.DOTALL | re.VERBOSE).fullmatch @@ -499,6 +503,7 @@ def _format_float_style(self, match): minimumwidth = int(match["minimumwidth"] or "0") thousands_sep = match["thousands_sep"] precision = int(match["precision"] or "6") + frac_sep = match["frac_separators"] or "" presentation_type = match["presentation_type"] trim_zeros = presentation_type in "gG" and not alternate_form trim_point = not alternate_form @@ -555,6 +560,9 @@ def _format_float_style(self, match): if trim_zeros: frac_part = frac_part.rstrip("0") separator = "" if trim_point and not frac_part else "." + if frac_sep: + frac_part = frac_sep.join(frac_part[pos:pos + 3] + for pos in range(0, len(frac_part), 3)) trailing = separator + frac_part + suffix # Do zero padding if required. diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 1875a2f529c957..cf42b86358dbca 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1322,6 +1322,8 @@ def test_format_e_presentation_type(self): # Thousands separators (F('1234567.123456'), ',.5e', '1.23457e+06'), (F('123.123456'), '012_.2e', '0_001.23e+02'), + # Thousands separators for fractional part (or for integral too) + (F('1234567.123456'), '.5_e', '1.234_57e+06'), # z flag is legal, but never makes a difference to the output (F(-1, 7**100), 'z.6e', '-3.091690e-85'), ] @@ -1447,6 +1449,12 @@ def test_format_f_presentation_type(self): (F('1234567'), ',.2f', '1,234,567.00'), (F('12345678'), ',.2f', '12,345,678.00'), (F('12345678'), ',f', '12,345,678.000000'), + # Thousands separators for fractional part (or for integral too) + (F('123456.789123123'), '._f', '123456.789_123'), + (F('123456.789123123'), '.7_f', '123456.789_123_1'), + (F('123456.789123123'), '.9_f', '123456.789_123_123'), + (F('123456.789123123'), '.,f', '123456.789,123'), + (F('123456.789123123'), '_.,f', '123_456.789,123'), # Underscore as thousands separator (F(2, 3), '_.2f', '0.67'), (F(2, 3), '_.7f', '0.6666667'), @@ -1620,6 +1628,11 @@ def test_invalid_formats(self): '.f', '.g', '.%', + # Thousands separators before precision + '._6e', + '._6f', + '._6g', + '._6%', # Z instead of z for negative zero suppression 'Z.2f' # z flag not supported for general formatting diff --git a/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst b/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst new file mode 100644 index 00000000000000..be2a30d69cab45 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst @@ -0,0 +1,2 @@ +Support underscore and comma as thousands separators in the fractional part +for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. From e25531331b8af9aff46986890826855e602c8201 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:24:30 +0200 Subject: [PATCH 576/713] [3.14] gh-87790: support thousands separators for formatting fractional part of Decimal (GH-132202) (#136365) Co-authored-by: Sergey B Kirpichev Co-authored-by: Serhiy Storchaka --- Lib/_pydecimal.py | 14 +++++++++++++- Lib/test/test_decimal.py | 13 +++++++++++++ .../2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst | 2 ++ 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 781b38ec26ba33..9b8e42a2342536 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -6122,7 +6122,11 @@ def _convert_for_comparison(self, other, equality_op=False): (?P0)? (?P\d+)? (?P[,_])? -(?:\.(?P\d+))? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) @@ -6215,6 +6219,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6334,6 +6341,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index ca2b0b18fb0a6d..c621b7ac08c3ac 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -1090,6 +1090,15 @@ def test_formatting(self): ('07_', '1234.56', '1_234.56'), ('_', '1.23456789', '1.23456789'), ('_%', '123.456789', '12_345.6789%'), + # and now for something completely different... + ('.,', '1.23456789', '1.234,567,89'), + ('._', '1.23456789', '1.234_567_89'), + ('.6_f', '12345.23456789', '12345.234_568'), + (',._%', '123.456789', '12,345.678_9%'), + (',._e', '123456', '1.234_56e+5'), + (',.4_e', '123456', '1.234_6e+5'), + (',.3_e', '123456', '1.235e+5'), + (',._E', '123456', '1.234_56E+5'), # negative zero: default behavior ('.1f', '-0', '-0.0'), @@ -1163,6 +1172,10 @@ def test_formatting(self): # bytes format argument self.assertRaises(TypeError, Decimal(1).__format__, b'-020') + # precision or fractional part separator should follow after dot + self.assertRaises(ValueError, format, Decimal(1), '.f') + self.assertRaises(ValueError, format, Decimal(1), '._6f') + def test_negative_zero_format_directed_rounding(self): with self.decimal.localcontext() as ctx: ctx.rounding = ROUND_CEILING diff --git a/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst b/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst new file mode 100644 index 00000000000000..cf80c71271bbd1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst @@ -0,0 +1,2 @@ +Support underscore and comma as thousands separators in the fractional part +for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. From 5a7d266fcbb35f772de4f20850710f03c06b8793 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 18:50:56 +0200 Subject: [PATCH 577/713] [3.14] gh-101100: Fix Sphinx warnings in library/email.compat32-message.rst (GH-136323) (#136389) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/library/email.compat32-message.rst | 18 +++++++++--------- Doc/tools/.nitignore | 1 - 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 4285c436e8da80..5754c2b65b239f 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: :meth:`set_payload` instead. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the related ``make`` and ``add`` methods. @@ -224,7 +224,7 @@ Here are the methods of the :class:`Message` class: ASCII charset. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.get_content` and :meth:`~email.message.EmailMessage.iter_parts`. @@ -236,7 +236,7 @@ Here are the methods of the :class:`Message` class: the message's default character set; see :meth:`set_charset` for details. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content`. @@ -265,9 +265,9 @@ Here are the methods of the :class:`Message` class: using that :mailheader:`Content-Transfer-Encoding` and is not modified. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. + :meth:`email.message.EmailMessage.set_content` method. .. method:: get_charset() @@ -276,7 +276,7 @@ Here are the methods of the :class:`Message` class: message's payload. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns + :class:`~email.message.EmailMessage` class it always returns ``None``. @@ -486,7 +486,7 @@ Here are the methods of the :class:`Message` class: search instead of :mailheader:`Content-Type`. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -524,7 +524,7 @@ Here are the methods of the :class:`Message` class: to ``False``. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -579,7 +579,7 @@ Here are the methods of the :class:`Message` class: header is also added. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the ``make_`` and ``add_`` methods. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 88f22db56f9684..eb760e0fac5e37 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -15,7 +15,6 @@ Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst -Doc/library/email.compat32-message.rst Doc/library/email.parser.rst Doc/library/exceptions.rst Doc/library/functools.rst From c72699086fe49ed127c1276e96027729381ea8c7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 7 Jul 2025 19:21:28 +0200 Subject: [PATCH 578/713] [3.14] gh-135906: Use `_PyObject_CAST` in internal headers (GH-135892) (GH-136392) gh-135906: Use `_PyObject_CAST` in internal headers (GH-135892) Fixes build errors encountered in python-greenlet/greenlet#450 when building greenlet on the free-threaded build. --------- (cherry picked from commit fe187fae8d8321f1b8d3c9560a35efe904de4217) Co-authored-by: Charlie Lin Co-authored-by: Peter Bierma Co-authored-by: Victor Stinner --- Include/internal/pycore_object.h | 4 ++-- Include/internal/pycore_stackref.h | 2 +- .../next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 1f9c7b1bf55b3f..12c5614834f565 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -614,7 +614,7 @@ static inline PyObject * _Py_XGetRef(PyObject **ptr) { for (;;) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -629,7 +629,7 @@ _Py_XGetRef(PyObject **ptr) static inline PyObject * _Py_TryXGetRef(PyObject **ptr) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 21306b95f9c2e4..0ce759fc743d82 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -765,7 +765,7 @@ _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) static inline int _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) { - PyObject *op = _Py_atomic_load_ptr_relaxed(src); + PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src)); if (op == NULL) { *out = PyStackRef_NULL; return 1; diff --git a/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst b/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst new file mode 100644 index 00000000000000..7852759a702804 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst @@ -0,0 +1 @@ +Fix compilation errors when compiling the internal headers with a C++ compiler. From e1e9d9535239537dcd4e161654fb324751a89ab4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 04:34:18 +0200 Subject: [PATCH 579/713] [3.14] gh-136032: Fix `argparse.BooleanOptionalAction` doc (GH-136133) (#136329) gh-136032: Fix `argparse.BooleanOptionalAction` doc (GH-136133) (cherry picked from commit 1953713d0d67a4f54ff75bf8449895a2f08cc750) Co-authored-by: W. H. Wang --- Doc/library/argparse.rst | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index a03d88092dbf1f..f189f6b8fa8953 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -839,23 +839,11 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -Only actions that consume command-line arguments (e.g. ``'store'``, -``'append'`` or ``'extend'``) can be used with positional arguments. - -.. class:: BooleanOptionalAction - - You may also specify an arbitrary action by passing an :class:`Action` subclass or - other object that implements the same interface. The :class:`!BooleanOptionalAction` - is available in :mod:`!argparse` and adds support for boolean actions such as - ``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - - .. versionadded:: 3.9 +You may also specify an arbitrary action by passing an :class:`Action` subclass +(e.g. :class:`BooleanOptionalAction`) or other object that implements the same +interface. Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'``, ``'extend'``, or custom actions with non-zero ``nargs``) can be used +with positional arguments. The recommended way to create a custom action is to extend :class:`Action`, overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and @@ -1429,6 +1417,21 @@ this API may be passed as the ``action`` parameter to and return a string which will be used when printing the usage of the program. If such method is not provided, a sensible default will be used. +.. class:: BooleanOptionalAction + + A subclass of :class:`Action` for handling boolean flags with positive + and negative options. Adding a single argument such as ``--foo`` automatically + creates both ``--foo`` and ``--no-foo`` options, storing ``True`` and ``False`` + respectively:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + + .. versionadded:: 3.9 + The parse_args() method ----------------------- From 78e5d109f54351bb2360a32e1cdcdd6a09b08791 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 08:03:03 +0200 Subject: [PATCH 580/713] [3.14] gh-135913: Document ob_refcnt, ob_type, ob_size (GH-135914) (GH-136377) gh-135913: Document ob_refcnt, ob_type, ob_size (GH-135914) * gh-135913: Document ob_refcnt, ob_type, ob_size In `typeobj.rst`, instead of `:c:member:` it would be better to use `.. c:member::` with a `:no-index:` option, see: See ref. https://www.sphinx-doc.org/en/master/usage/domains/index.html#basic-markup However, `c:member` currently does not support `:no-index:`. (cherry picked from commit 73e1207a4ebdb3b43d597cd6c288dae6d7d1dbdb) Co-authored-by: Petr Viktorin --- Doc/c-api/structures.rst | 75 +++++++++++++++++++++++++++++++--------- Doc/c-api/typeobj.rst | 11 +++--- 2 files changed, 64 insertions(+), 22 deletions(-) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 987bc167c68d6c..58dd915e04f619 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -28,18 +28,52 @@ under :ref:`reference counting `. object. In a normal "release" build, it contains only the object's reference count and a pointer to the corresponding type object. Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + to a Python object can be cast to a :c:expr:`PyObject*`. + + The members must not be accessed directly; instead use macros such as + :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_refcnt + + The object's reference count, as returned by :c:macro:`Py_REFCNT`. + Do not use this field directly; instead use functions and macros such as + :c:macro:`!Py_REFCNT`, :c:func:`Py_INCREF` and :c:func:`Py_DecRef`. + + The field type may be different from ``Py_ssize_t``, depending on + build configuration and platform. + + .. c:member:: PyTypeObject* ob_type + + The object's type. + Do not use this field directly; use :c:macro:`Py_TYPE` and + :c:func:`Py_SET_TYPE` instead. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + An extension of :c:type:`PyObject` that adds the + :c:member:`~PyVarObject.ob_size` field. + This is intended for objects that have some notion of *length*. + + As with :c:type:`!PyObject`, the members must not be accessed directly; + instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and + :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_size + + A size field, whose contents should be considered an object's internal + implementation detail. + + Do not use this field directly; use :c:macro:`Py_SIZE` instead. + + Object creation functions such as :c:func:`PyObject_NewVar` will + generally set this field to the requested size (number of items). + After creation, arbitrary values can be stored in :c:member:`!ob_size` + using :c:macro:`Py_SET_SIZE`. + + To get an object's publicly exposed length, as returned by + the Python function :py:func:`len`, use :c:func:`PyObject_Length` + instead. .. c:macro:: PyObject_HEAD @@ -103,9 +137,8 @@ under :ref:`reference counting `. Get the type of the Python object *o*. - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. + The returned reference is :term:`borrowed ` from *o*. + Do not release it with :c:func:`Py_DECREF` or similar. .. versionchanged:: 3.11 :c:func:`Py_TYPE()` is changed to an inline static function. @@ -122,16 +155,26 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - Set the object *o* type to *type*. + Set the type of object *o* to *type*, without any checking or reference + counting. + + This is a very low-level operation. + Consider instead setting the Python attribute :attr:`~object.__class__` + using :c:func:`PyObject_SetAttrString` or similar. + + Note that assigning an incompatible type can lead to undefined behavior. + + If *type* is a :ref:`heap type `, the caller must create a + new reference to it. + Similarly, if the old type of *o* is a heap type, the caller must release + a reference to that type. .. versionadded:: 3.9 .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. + Get the :c:member:`~PyVarObject.ob_size` field of *o*. .. versionchanged:: 3.11 :c:func:`Py_SIZE()` is changed to an inline static function. @@ -140,7 +183,7 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size to *size*. + Set the :c:member:`~PyVarObject.ob_size` field of *o* to *size*. .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index af2bead3bb5004..060d6f60174b41 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -492,9 +492,9 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. -.. c:member:: Py_ssize_t PyObject.ob_refcnt +:c:member:`PyObject.ob_refcnt` - This is the type object's reference count, initialized to ``1`` by the + The type object's reference count is initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for @@ -506,7 +506,7 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is not inherited by subtypes. -.. c:member:: PyTypeObject* PyObject.ob_type +:c:member:`PyObject.ob_type` This is the type's type, in other words its metatype. It is initialized by the argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be @@ -532,14 +532,13 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. PyVarObject Slots ----------------- -.. c:member:: Py_ssize_t PyVarObject.ob_size +:c:member:`PyVarObject.ob_size` For :ref:`statically allocated type objects `, this should be initialized to zero. For :ref:`dynamically allocated type objects `, this field has a special internal meaning. - This field should be accessed using the :c:func:`Py_SIZE()` and - :c:func:`Py_SET_SIZE()` macros. + This field should be accessed using the :c:func:`Py_SIZE()` macro. **Inheritance:** From cdeaa64214ab154b43ddb8d9b97b28f3e3459cec Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 08:24:08 +0200 Subject: [PATCH 581/713] [3.14] gh-102567: Add missing newline to `--help-all` (GH-136391) (GH-136403) gh-102567: Add missing newline to `--help-all` (GH-136391) (cherry picked from commit fbef0c1d6a6540f5063138b861f79bdb2a935b0a) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Python/initconfig.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/initconfig.c b/Python/initconfig.c index 25e30aa648e8aa..62bcb727d1b9bf 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -312,7 +312,7 @@ The following implementation-specific options are available:\n\ "-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" #endif "\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\ +-X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ From 2733b290ea74e073ded5fef9b825f40185d0db44 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 08:47:17 +0200 Subject: [PATCH 582/713] [3.14] gh-101100: Fix sphinx warnings in `whatsnew/3.11.rst` (GH-136402) (#136405) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/tools/.nitignore | 1 - Doc/whatsnew/3.11.rst | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index eb760e0fac5e37..e29775b075a82f 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -73,4 +73,3 @@ Doc/whatsnew/3.6.rst Doc/whatsnew/3.7.rst Doc/whatsnew/3.8.rst Doc/whatsnew/3.10.rst -Doc/whatsnew/3.11.rst diff --git a/Doc/whatsnew/3.11.rst b/Doc/whatsnew/3.11.rst index 2dd205dd2b8831..abf9677fd9cac5 100644 --- a/Doc/whatsnew/3.11.rst +++ b/Doc/whatsnew/3.11.rst @@ -1292,7 +1292,7 @@ This section covers specific optimizations independent of the (Contributed by Stefan Behnel in :gh:`68264`.) * Resizing lists is streamlined for the common case, - speeding up :meth:`list.append` by ≈15% + speeding up :meth:`!list.append` by ≈15% and simple :term:`list comprehension`\s by up to 20-30% (Contributed by Dennis Sweeney in :gh:`91165`.) From 3296d9bcfd51d4df58d437264182ee421b8cd2ca Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 09:03:40 +0200 Subject: [PATCH 583/713] [3.14] gh-136155: Docs: only add custom OpenGraph protocol meta tags for HTML builds (GH-136187) (#136382) Co-authored-by: Maciej Olko Co-authored-by: blurb-it[bot] <43283697+blurb-it[bot]@users.noreply.github.com> --- Doc/conf.py | 19 ++++++++++--------- ...-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst | 1 + 2 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst diff --git a/Doc/conf.py b/Doc/conf.py index 161c2986441edd..8b2a8f20fcc558 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -635,13 +635,14 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -ogp_custom_meta_tags = [ - '', -] -if 'create-social-cards' not in tags: # noqa: F821 - # Define a static preview image when not creating social cards - ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ - '', - '', +if 'builder_html' in tags: # noqa: F821 + ogp_custom_meta_tags = [ + '', ] + if 'create-social-cards' not in tags: # noqa: F821 + # Define a static preview image when not creating social cards + ogp_image = '_static/og-image.png' + ogp_custom_meta_tags += [ + '', + '', + ] diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst new file mode 100644 index 00000000000000..0341b5f7f0d5e6 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst @@ -0,0 +1 @@ +EPUB builds are fixed by excluding non-XHTML-compatible tags. From 145c08629bc2df2176cd03b69215e652e3f946c4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:16:51 +0200 Subject: [PATCH 584/713] [3.14] gh-136297: Fix `hypothesis` and `subTest` usage in `test_zoneinfo_property.py` (GH-136384) (#136407) gh-136297: Fix `hypothesis` and `subTest` usage in `test_zoneinfo_property.py` (GH-136384) (cherry picked from commit db699db99d214dccb7c4849fa609fdd5188ee038) Co-authored-by: sobolevn --- Lib/test/test_zoneinfo/test_zoneinfo_property.py | 16 +++++++--------- Tools/requirements-hypothesis.txt | 2 +- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index 294c7e9b27a857..c00815e2fd4c36 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -147,23 +147,21 @@ def setUp(self): def test_pickle_unpickle_cache(self, key): zi = self.klass(key) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - pkl_str = pickle.dumps(zi, proto) - zi_rt = pickle.loads(pkl_str) + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIs(zi, zi_rt) + self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.subTest(proto=proto): - pkl_str = pickle.dumps(zi, proto) - zi_rt = pickle.loads(pkl_str) + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIsNot(zi, zi_rt) - self.assertEqual(str(zi), str(zi_rt)) + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) @add_key_examples diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 66898885c0a412..e5deac497fbe3f 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.111.2 +hypothesis==6.135.26 From 376e037eccdb7a75c28725b470bd7f916a9bb987 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 10:43:57 +0200 Subject: [PATCH 585/713] [3.14] gh-120713: Make _Py_NORMALIZE_CENTURY private (GH-135933) (#136387) Co-authored-by: Petr Viktorin --- Modules/_datetimemodule.c | 2 +- configure | 2 +- configure.ac | 2 +- pyconfig.h.in | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26b..f5a9712b64e4de 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -1928,7 +1928,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } replacement = freplacement; } -#ifdef Py_NORMALIZE_CENTURY +#ifdef _Py_NORMALIZE_CENTURY else if (ch == 'Y' || ch == 'G' || ch == 'F' || ch == 'C' ) { diff --git a/configure b/configure index a9257ac0b3ac30..1646efe5f061ca 100755 --- a/configure +++ b/configure @@ -28234,7 +28234,7 @@ printf "%s\n" "$ac_cv_normalize_century" >&6; } if test "$ac_cv_normalize_century" = yes then -printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h +printf "%s\n" "#define _Py_NORMALIZE_CENTURY 1" >>confdefs.h fi diff --git a/configure.ac b/configure.ac index 225dfc79ab5b29..d868f7b26bad1c 100644 --- a/configure.ac +++ b/configure.ac @@ -6813,7 +6813,7 @@ int main(void) [ac_cv_normalize_century=yes])]) if test "$ac_cv_normalize_century" = yes then - AC_DEFINE([Py_NORMALIZE_CENTURY], [1], + AC_DEFINE([_Py_NORMALIZE_CENTURY], [1], [Define if year with century should be normalized for strftime.]) fi diff --git a/pyconfig.h.in b/pyconfig.h.in index d7c496fccc682c..1c533b2bfb7fb4 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1737,9 +1737,6 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM -/* Define if year with century should be normalized for strftime. */ -#undef Py_NORMALIZE_CENTURY - /* Define if you want to enable remote debugging support. */ #undef Py_REMOTE_DEBUG @@ -2026,6 +2023,9 @@ /* HACL* library can compile SIMD256 implementations */ #undef _Py_HACL_CAN_COMPILE_VEC256 +/* Define if year with century should be normalized for strftime. */ +#undef _Py_NORMALIZE_CENTURY + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT From 7ec1faba0c85d28ae490555f0922c86eb50e874b Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 8 Jul 2025 11:56:46 +0300 Subject: [PATCH 586/713] Python 3.14.0b4 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 373 ++++++++++++-- Misc/NEWS.d/3.14.0b4.rst | 484 ++++++++++++++++++ ...-05-19-18-09-20.gh-issue-134273.ZAliyy.rst | 1 - ...-06-25-01-03-10.gh-issue-135906.UBrCWq.rst | 1 - ...-05-31-19-24-54.gh-issue-134280.NDVbzY.rst | 2 - ...-06-16-02-31-42.gh-issue-135543.6b0HOF.rst | 2 - ...-06-17-12-50-48.gh-issue-135608.PnHckD.rst | 1 - ...-06-17-22-34-58.gh-issue-135607.ucsLVu.rst | 2 - ...-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst | 2 - ...-06-23-18-08-32.gh-issue-135871.50C528.rst | 2 - ...-06-24-06-41-47.gh-issue-129958.EaJuS0.rst | 2 - ...5-06-26-15-25-51.gh-issue-78465.MbDN8X.rst | 2 - ...-07-06-14-53-19.gh-issue-109700.KVNQQi.rst | 1 - ...-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst | 1 - ...-03-11-05-24-14.gh-issue-130664.g0yNMm.rst | 4 - ...5-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst | 2 - ...5-04-07-10-20-16.gh-issue-87790.X2SjJe.rst | 2 - ...-06-02-14-28-30.gh-issue-130662.EIgIR8.rst | 3 - ...-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst | 3 - ...-06-03-12-59-17.gh-issue-135069.xop30V.rst | 3 - ...-06-10-21-42-04.gh-issue-135335.WnUqb_.rst | 2 - ...-06-14-12-06-55.gh-issue-135487.KdVFff.rst | 2 - ...-06-16-15-03-03.gh-issue-135561.mJCN8D.rst | 2 - ...-06-17-22-44-19.gh-issue-119180.Ogv8Nj.rst | 2 - ...-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst | 2 - ...-06-18-11-43-17.gh-issue-135646.r7ekEn.rst | 1 - ...-06-18-13-58-13.gh-issue-135645.109nff.rst | 2 - ...-06-22-02-16-17.gh-issue-135640.FXyFL6.rst | 4 - ...-06-22-16-23-44.gh-issue-135815.0DandH.rst | 2 - ...-06-23-10-19-11.gh-issue-135855.-J0AGF.rst | 3 - ...-06-23-11-04-25.gh-issue-135836.-C-c4v.rst | 1 - ...-06-24-10-52-35.gh-issue-135836.s37351.rst | 3 - ...-06-24-14-43-24.gh-issue-135878.Db4roX.rst | 3 - ...5-06-25-17-25-53.gh-issue-91555.xUpTLD.rst | 4 - ...5-06-26-11-52-40.gh-issue-53203.TMigBr.rst | 2 - ...-06-26-17-28-49.gh-issue-135995.pPrDCt.rst | 1 - ...-06-27-13-34-28.gh-issue-136028.RY727g.rst | 3 - ...5-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst | 3 - ...-07-05-06-56-16.gh-issue-136316.3zj_Do.rst | 2 - ...-07-05-09-45-04.gh-issue-136286.N67Amr.rst | 2 - ...-06-18-13-28-08.gh-issue-102555.nADrzJ.rst | 3 - ...-06-25-14-13-39.gh-issue-135661.idjQ0B.rst | 25 - ...-06-27-21-23-19.gh-issue-136053.QZxcee.rst | 1 - ...-06-19-15-29-38.gh-issue-135494.FVl9a0.rst | 2 - ...-06-26-15-15-35.gh-issue-135966.EBpF8Y.rst | 1 - ...-06-25-10-36-22.gh-issue-133600.bkdgHC.rst | 4 - ...-06-26-15-58-13.gh-issue-135968.C4v_-W.rst | 1 - README.rst | 2 +- 49 files changed, 806 insertions(+), 176 deletions(-) create mode 100644 Misc/NEWS.d/3.14.0b4.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-16-02-31-42.gh-issue-135543.6b0HOF.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-12-50-48.gh-issue-135608.PnHckD.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-18-08-32.gh-issue-135871.50C528.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-06-41-47.gh-issue-129958.EaJuS0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-26-15-25-51.gh-issue-78465.MbDN8X.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-10-21-42-04.gh-issue-135335.WnUqb_.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-14-12-06-55.gh-issue-135487.KdVFff.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-16-15-03-03.gh-issue-135561.mJCN8D.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-17-22-44-19.gh-issue-119180.Ogv8Nj.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-18-11-43-17.gh-issue-135646.r7ekEn.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-18-13-58-13.gh-issue-135645.109nff.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-22-16-23-44.gh-issue-135815.0DandH.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-23-10-19-11.gh-issue-135855.-J0AGF.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-24-14-43-24.gh-issue-135878.Db4roX.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-25-17-25-53.gh-issue-91555.xUpTLD.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-26-11-52-40.gh-issue-53203.TMigBr.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-26-17-28-49.gh-issue-135995.pPrDCt.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-27-13-34-28.gh-issue-136028.RY727g.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-06-19-15-29-38.gh-issue-135494.FVl9a0.rst delete mode 100644 Misc/NEWS.d/next/Tests/2025-06-26-15-15-35.gh-issue-135966.EBpF8Y.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-06-25-10-36-22.gh-issue-133600.bkdgHC.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-06-26-15-58-13.gh-issue-135968.C4v_-W.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index ad88a909b105ef..40135791403f3c 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -21,10 +21,10 @@ #define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 3 +#define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.14.0b3+" +#define PY_VERSION "3.14.0b4" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 2376d6e70e1743..e32517960d44d1 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Jun 17 18:40:47 2025 +# Autogenerated by Sphinx on Tue Jul 8 11:57:16 2025 # as part of the release process. topics = { @@ -492,8 +492,7 @@ async def func(param1, param2): Python supports string and bytes literals and various numeric literals: - literal: stringliteral | bytesliteral - | integer | floatnumber | imagnumber + literal: stringliteral | bytesliteral | NUMBER Evaluation of a literal yields an object of the given type (string, bytes, integer, floating-point number, complex number) with the given @@ -5132,25 +5131,54 @@ class of the instance or a *non-virtual base class* thereof. The 'floating': r'''Floating-point literals *********************** -Floating-point literals are described by the following lexical -definitions: +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed in the numeric +parts. For example, "077.010" is legal, and denotes the same number as +"77.10". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: - floatnumber: pointfloat | exponentfloat - pointfloat: [digitpart] fraction | digitpart "." - exponentfloat: (digitpart | pointfloat) exponent - digitpart: digit (["_"] digit)* - fraction: "." digitpart - exponent: ("e" | "E") ["+" | "-"] digitpart + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) -Note that the integer and exponent parts are always interpreted using -radix 10. For example, "077e010" is legal, and denotes the same number -as "77e10". The allowed range of floating-point literals is -implementation-dependent. As in integer literals, underscores are -supported for digit grouping. +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: -Some examples of floating-point literals: + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. @@ -6145,17 +6173,53 @@ class body. A "SyntaxError" is raised if a variable is used or 'imaginary': r'''Imaginary literals ****************** -Imaginary literals are described by the following lexical definitions: +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. - imagnumber: (floatnumber | digitpart) ("j" | "J") +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j -An imaginary literal yields a complex number with a real part of 0.0. -Complex numbers are represented as a pair of floating-point numbers -and have the same restrictions on their range. To create a complex -number with a nonzero real part, add a floating-point number to it, -e.g., "(3+4j)". Some examples of imaginary literals: +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") ''', 'import': r'''The "import" statement ********************** @@ -6397,37 +6461,62 @@ class body. A "SyntaxError" is raised if a variable is used or 'integers': r'''Integer literals **************** -Integer literals are described by the following lexical definitions: +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: - integer: decinteger | bininteger | octinteger | hexinteger - decinteger: nonzerodigit (["_"] digit)* | "0"+ (["_"] "0")* + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* bininteger: "0" ("b" | "B") (["_"] bindigit)+ octinteger: "0" ("o" | "O") (["_"] octdigit)+ hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* nonzerodigit: "1"..."9" digit: "0"..."9" bindigit: "0" | "1" octdigit: "0"..."7" hexdigit: digit | "a"..."f" | "A"..."F" -There is no limit for the length of integer literals apart from what -can be stored in available memory. - -Underscores are ignored for determining the numeric value of the -literal. They can be used to group digits for enhanced readability. -One underscore can occur between digits, and after base specifiers -like "0x". - -Note that leading zeros in a non-zero decimal number are not allowed. -This is for disambiguation with C-style octal literals, which Python -used before version 3.0. - -Some examples of integer literals: - - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 - Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. ''', @@ -6774,14 +6863,190 @@ class body. A "SyntaxError" is raised if a variable is used or 'numbers': r'''Numeric literals **************** -There are three types of numeric literals: integers, floating-point -numbers, and imaginary numbers. There are no complex literals -(complex numbers can be formed by adding a real number and an -imaginary number). +"NUMBER" tokens represent numeric literals, of which there are three +types: integers, floating-point numbers, and imaginary numbers. + + NUMBER: integer | floatnumber | imagnumber + +The numeric value of a numeric literal is the same as if it were +passed as a string to the "int", "float" or "complex" class +constructor, respectively. Note that not all valid inputs for those +constructors are also valid literals. + +Numeric literals do not include a sign; a phrase like "-1" is actually +an expression composed of the unary operator ‘"-"’ and the literal +"1". + + +Integer literals +================ + +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 -Note that numeric literals do not include a sign; a phrase like "-1" -is actually an expression composed of the unary operator ‘"-"’ and the -literal "1". +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Floating-point literals +======================= + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed in the numeric +parts. For example, "077.010" is legal, and denotes the same number as +"77.10". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Imaginary literals +================== + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") ''', 'numeric-types': r'''Emulating numeric types *********************** diff --git a/Misc/NEWS.d/3.14.0b4.rst b/Misc/NEWS.d/3.14.0b4.rst new file mode 100644 index 00000000000000..b96f96caa3f280 --- /dev/null +++ b/Misc/NEWS.d/3.14.0b4.rst @@ -0,0 +1,484 @@ +.. date: 2025-06-26-15-58-13 +.. gh-issue: 135968 +.. nonce: C4v_-W +.. release date: 2025-07-08 +.. section: Tools/Demos + +Stubs for ``strip`` are now provided as part of an iOS install. + +.. + +.. date: 2025-06-25-10-36-22 +.. gh-issue: 133600 +.. nonce: bkdgHC +.. section: Tools/Demos + +Backport file reorganization for Tools/wasm/wasi. + +This should make backporting future code changes easier. It also simplifies +instructions around how to do WASI builds in the devguide. + +.. + +.. date: 2025-06-26-15-15-35 +.. gh-issue: 135966 +.. nonce: EBpF8Y +.. section: Tests + +The iOS testbed now handles the ``app_packages`` folder as a site directory. + +.. + +.. date: 2025-06-19-15-29-38 +.. gh-issue: 135494 +.. nonce: FVl9a0 +.. section: Tests + +Fix regrtest to support excluding tests from ``--pgo`` tests. Patch by +Victor Stinner. + +.. + +.. date: 2025-06-27-21-23-19 +.. gh-issue: 136053 +.. nonce: QZxcee +.. section: Security + +:mod:`marshal`: fix a possible crash when deserializing :class:`slice` +objects. + +.. + +.. date: 2025-06-25-14-13-39 +.. gh-issue: 135661 +.. nonce: idjQ0B +.. section: Security + +Fix parsing start and end tags in :class:`html.parser.HTMLParser` according +to the HTML5 standard. + +* Whitespaces no longer accepted between ```` does not end the script section. + +* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized + as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. + +* Null character (U+0000) no longer ends the tag name. + +* Attributes and slashes after the tag name in end tags are now ignored, + instead of terminating after the first ``>`` in quoted attribute value. + E.g. ````. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ````. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ```` produces attribute "foo" with value "=bar". + +* Whitespaces between the ``=`` separator and attribute name or value are no + longer ignored. E.g. ```` produces two attributes "foo" and + "=bar", both with value None; ```` produces two attributes: + "foo" with value "" and "bar" with value None. + +.. + +.. date: 2025-06-18-13-28-08 +.. gh-issue: 102555 +.. nonce: nADrzJ +.. section: Security + +Fix comment parsing in :class:`html.parser.HTMLParser` according to the +HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. + +.. + +.. date: 2025-07-05-09-45-04 +.. gh-issue: 136286 +.. nonce: N67Amr +.. section: Library + +Fix pickling failures for protocols 0 and 1 for many objects realted to +subinterpreters. + +.. + +.. date: 2025-07-05-06-56-16 +.. gh-issue: 136316 +.. nonce: 3zj_Do +.. section: Library + +Improve support for evaluating nested forward references in +:func:`typing.evaluate_forward_ref`. + +.. + +.. date: 2025-06-30-11-12-24 +.. gh-issue: 85702 +.. nonce: 0Lrbwu +.. section: Library + +If ``zoneinfo._common.load_tzdata`` is given a package without a resource a +:exc:`zoneinfo.ZoneInfoNotFoundError` is raised rather than a +:exc:`PermissionError`. Patch by Victor Stinner. + +.. + +.. date: 2025-06-27-13-34-28 +.. gh-issue: 136028 +.. nonce: RY727g +.. section: Library + +Fix parsing month names containing "İ" (U+0130, LATIN CAPITAL LETTER I WITH +DOT ABOVE) in :func:`time.strptime`. This affects locales az_AZ, ber_DZ, +ber_MA and crh_UA. + +.. + +.. date: 2025-06-26-17-28-49 +.. gh-issue: 135995 +.. nonce: pPrDCt +.. section: Library + +In the palmos encoding, make byte ``0x9b`` decode to ``›`` (U+203A - SINGLE +RIGHT-POINTING ANGLE QUOTATION MARK). + +.. + +.. date: 2025-06-26-11-52-40 +.. gh-issue: 53203 +.. nonce: TMigBr +.. section: Library + +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats on locales byn_ER, +wal_ET and lzh_TW, and for ``%X`` format on locales ar_SA, bg_BG and lzh_TW. + +.. + +.. date: 2025-06-25-17-25-53 +.. gh-issue: 91555 +.. nonce: xUpTLD +.. section: Library + +An earlier change, which was introduced in 3.14.0b2, has been reverted. It +disabled logging for a logger during handling of log messages for that +logger. Since the reversion, the behaviour should be as it was before +3.14.0b2. + +.. + +.. date: 2025-06-24-14-43-24 +.. gh-issue: 135878 +.. nonce: Db4roX +.. section: Library + +Fixes a crash of :class:`types.SimpleNamespace` on :term:`free threading` +builds, when several threads were calling its :meth:`~object.__repr__` +method at the same time. + +.. + +.. date: 2025-06-24-10-52-35 +.. gh-issue: 135836 +.. nonce: s37351 +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when non-\ :exc:`OSError` exception is raised during connection and +socket's ``close()`` raises :exc:`!OSError`. + +.. + +.. date: 2025-06-23-11-04-25 +.. gh-issue: 135836 +.. nonce: -C-c4v +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when the Happy Eyeballs algorithm resulted in an empty exceptions list +during connection attempts. + +.. + +.. date: 2025-06-23-10-19-11 +.. gh-issue: 135855 +.. nonce: -J0AGF +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`SystemError` when +:func:`!_interpreters.set___main___attrs` is passed a non-dict object. Patch +by Brian Schubert. + +.. + +.. date: 2025-06-22-16-23-44 +.. gh-issue: 135815 +.. nonce: 0DandH +.. section: Library + +:mod:`netrc`: skip security checks if :func:`os.getuid` is missing. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-22-02-16-17 +.. gh-issue: 135640 +.. nonce: FXyFL6 +.. section: Library + +Address bug where it was possible to call +:func:`xml.etree.ElementTree.ElementTree.write` on an ElementTree object +with an invalid root element. This behavior blanked the file passed to +``write`` if it already existed. + +.. + +.. date: 2025-06-18-13-58-13 +.. gh-issue: 135645 +.. nonce: 109nff +.. section: Library + +Added ``supports_isolated_interpreters`` field to +:data:`sys.implementation`. + +.. + +.. date: 2025-06-18-11-43-17 +.. gh-issue: 135646 +.. nonce: r7ekEn +.. section: Library + +Raise consistent :exc:`NameError` exceptions in +:func:`annotationlib.ForwardRef.evaluate` + +.. + +.. date: 2025-06-17-23-13-56 +.. gh-issue: 135557 +.. nonce: Bfcy4v +.. section: Library + +Fix races on :mod:`heapq` updates and :class:`list` reads on the :term:`free +threaded ` build. + +.. + +.. date: 2025-06-17-22-44-19 +.. gh-issue: 119180 +.. nonce: Ogv8Nj +.. section: Library + +Only fetch globals and locals if necessary in +:func:`annotationlib.get_annotations` + +.. + +.. date: 2025-06-16-15-03-03 +.. gh-issue: 135561 +.. nonce: mJCN8D +.. section: Library + +Fix a crash on DEBUG builds when an HACL* HMAC routine fails. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-14-12-06-55 +.. gh-issue: 135487 +.. nonce: KdVFff +.. section: Library + +Fix :meth:`!reprlib.Repr.repr_int` when given integers with more than +:func:`sys.get_int_max_str_digits` digits. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-10-21-42-04 +.. gh-issue: 135335 +.. nonce: WnUqb_ +.. section: Library + +:mod:`multiprocessing`: Flush ``stdout`` and ``stderr`` after preloading +modules in the ``forkserver``. + +.. + +.. date: 2025-06-03-12-59-17 +.. gh-issue: 135069 +.. nonce: xop30V +.. section: Library + +Fix the "Invalid error handling" exception in +:class:`!encodings.idna.IncrementalDecoder` to correctly replace the +'errors' parameter. + +.. + +.. date: 2025-06-02-14-36-28 +.. gh-issue: 130662 +.. nonce: Gpr2GB +.. section: Library + ++Accept leading zeros in precision and width fields for ++:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), +'.016f')``. + +.. + +.. date: 2025-06-02-14-28-30 +.. gh-issue: 130662 +.. nonce: EIgIR8 +.. section: Library + +Accept leading zeros in precision and width fields for +:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, +3), '.016f')``. + +.. + +.. date: 2025-04-07-10-20-16 +.. gh-issue: 87790 +.. nonce: X2SjJe +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-04-07-09-53-54 +.. gh-issue: 87790 +.. nonce: 6nj3zQ +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-03-11-05-24-14 +.. gh-issue: 130664 +.. nonce: g0yNMm +.. section: Library + +Handle corner-case for :class:`~fractions.Fraction`'s formatting: treat +zero-padding (preceding the width field by a zero (``'0'``) character) as an +equivalent to a fill character of ``'0'`` with an alignment type of ``'='``, +just as in case of :class:`float`'s. + +.. + +.. date: 2025-07-01-21-04-47 +.. gh-issue: 136155 +.. nonce: ufmH4Q +.. section: Documentation + +EPUB builds are fixed by excluding non-XHTML-compatible tags. + +.. + +.. date: 2025-07-06-14-53-19 +.. gh-issue: 109700 +.. nonce: KVNQQi +.. section: Core and Builtins + +Fix memory error handling in :c:func:`PyDict_SetDefault`. + +.. + +.. date: 2025-06-26-15-25-51 +.. gh-issue: 78465 +.. nonce: MbDN8X +.. section: Core and Builtins + +Fix error message for ``cls.__new__(cls, ...)`` where ``cls`` is not +instantiable builtin or extension type (with ``tp_new`` set to ``NULL``). + +.. + +.. date: 2025-06-24-06-41-47 +.. gh-issue: 129958 +.. nonce: EaJuS0 +.. section: Core and Builtins + +Differentiate between t-strings and f-strings in syntax error for newlines +in format specifiers of single-quoted interpolated strings. + +.. + +.. date: 2025-06-23-18-08-32 +.. gh-issue: 135871 +.. nonce: 50C528 +.. section: Core and Builtins + +Non-blocking mutex lock attempts now return immediately when the lock is +busy instead of briefly spinning in the :term:`free threading` build. + +.. + +.. date: 2025-06-18-16-45-36 +.. gh-issue: 135106 +.. nonce: cpl6Aq +.. section: Core and Builtins + +Restrict the trashcan mechanism to GC'ed objects and untrack them while in +the trashcan to prevent the GC and trashcan mechanisms conflicting. + +.. + +.. date: 2025-06-17-22-34-58 +.. gh-issue: 135607 +.. nonce: ucsLVu +.. section: Core and Builtins + +Fix potential :mod:`weakref` races in an object's destructor on the +:term:`free threaded ` build. + +.. + +.. date: 2025-06-17-12-50-48 +.. gh-issue: 135608 +.. nonce: PnHckD +.. section: Core and Builtins + +Fix a crash in the JIT involving attributes of modules. + +.. + +.. date: 2025-06-16-02-31-42 +.. gh-issue: 135543 +.. nonce: 6b0HOF +.. section: Core and Builtins + +Emit ``sys.remote_exec`` audit event when :func:`sys.remote_exec` is called +and migrate ``remote_debugger_script`` to +``cpython.remote_debugger_script``. + +.. + +.. date: 2025-05-31-19-24-54 +.. gh-issue: 134280 +.. nonce: NDVbzY +.. section: Core and Builtins + +Disable constant folding for ``~`` with a boolean argument. This moves the +deprecation warning from compile time to runtime. + +.. + +.. date: 2025-06-25-01-03-10 +.. gh-issue: 135906 +.. nonce: UBrCWq +.. section: C API + +Fix compilation errors when compiling the internal headers with a C++ +compiler. + +.. + +.. date: 2025-05-19-18-09-20 +.. gh-issue: 134273 +.. nonce: ZAliyy +.. section: Build + +Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` diff --git a/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst b/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst deleted file mode 100644 index 3eb13cefbe6dfc..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-05-19-18-09-20.gh-issue-134273.ZAliyy.rst +++ /dev/null @@ -1 +0,0 @@ -Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` diff --git a/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst b/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst deleted file mode 100644 index 7852759a702804..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-06-25-01-03-10.gh-issue-135906.UBrCWq.rst +++ /dev/null @@ -1 +0,0 @@ -Fix compilation errors when compiling the internal headers with a C++ compiler. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst deleted file mode 100644 index f822721690975a..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-31-19-24-54.gh-issue-134280.NDVbzY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Disable constant folding for ``~`` with a boolean argument. -This moves the deprecation warning from compile time to runtime. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-16-02-31-42.gh-issue-135543.6b0HOF.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-16-02-31-42.gh-issue-135543.6b0HOF.rst deleted file mode 100644 index 6efe2a47bac5d4..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-16-02-31-42.gh-issue-135543.6b0HOF.rst +++ /dev/null @@ -1,2 +0,0 @@ -Emit ``sys.remote_exec`` audit event when :func:`sys.remote_exec` is called -and migrate ``remote_debugger_script`` to ``cpython.remote_debugger_script``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-12-50-48.gh-issue-135608.PnHckD.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-12-50-48.gh-issue-135608.PnHckD.rst deleted file mode 100644 index a65a0c85fa64fa..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-12-50-48.gh-issue-135608.PnHckD.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash in the JIT involving attributes of modules. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst deleted file mode 100644 index 859259a9ace758..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-17-22-34-58.gh-issue-135607.ucsLVu.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix potential :mod:`weakref` races in an object's destructor on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst deleted file mode 100644 index b6e953a7719be1..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-18-16-45-36.gh-issue-135106.cpl6Aq.rst +++ /dev/null @@ -1,2 +0,0 @@ -Restrict the trashcan mechanism to GC'ed objects and untrack them while in -the trashcan to prevent the GC and trashcan mechanisms conflicting. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-18-08-32.gh-issue-135871.50C528.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-18-08-32.gh-issue-135871.50C528.rst deleted file mode 100644 index ce29ddecefe17f..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-23-18-08-32.gh-issue-135871.50C528.rst +++ /dev/null @@ -1,2 +0,0 @@ -Non-blocking mutex lock attempts now return immediately when the lock is busy -instead of briefly spinning in the :term:`free threading` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-06-41-47.gh-issue-129958.EaJuS0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-06-41-47.gh-issue-129958.EaJuS0.rst deleted file mode 100644 index 70b3e99425df14..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-24-06-41-47.gh-issue-129958.EaJuS0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Differentiate between t-strings and f-strings in syntax error for newlines -in format specifiers of single-quoted interpolated strings. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-26-15-25-51.gh-issue-78465.MbDN8X.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-26-15-25-51.gh-issue-78465.MbDN8X.rst deleted file mode 100644 index 99734d63c5d87e..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-26-15-25-51.gh-issue-78465.MbDN8X.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix error message for ``cls.__new__(cls, ...)`` where ``cls`` is not -instantiable builtin or extension type (with ``tp_new`` set to ``NULL``). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst deleted file mode 100644 index a37f4a51050947..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-06-14-53-19.gh-issue-109700.KVNQQi.rst +++ /dev/null @@ -1 +0,0 @@ -Fix memory error handling in :c:func:`PyDict_SetDefault`. diff --git a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst b/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst deleted file mode 100644 index 0341b5f7f0d5e6..00000000000000 --- a/Misc/NEWS.d/next/Documentation/2025-07-01-21-04-47.gh-issue-136155.ufmH4Q.rst +++ /dev/null @@ -1 +0,0 @@ -EPUB builds are fixed by excluding non-XHTML-compatible tags. diff --git a/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst b/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst deleted file mode 100644 index dbe783a2a99e0a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-03-11-05-24-14.gh-issue-130664.g0yNMm.rst +++ /dev/null @@ -1,4 +0,0 @@ -Handle corner-case for :class:`~fractions.Fraction`'s formatting: treat -zero-padding (preceding the width field by a zero (``'0'``) character) as an -equivalent to a fill character of ``'0'`` with an alignment type of ``'='``, -just as in case of :class:`float`'s. diff --git a/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst b/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst deleted file mode 100644 index cf80c71271bbd1..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-07-09-53-54.gh-issue-87790.6nj3zQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support underscore and comma as thousands separators in the fractional part -for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst b/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst deleted file mode 100644 index be2a30d69cab45..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-04-07-10-20-16.gh-issue-87790.X2SjJe.rst +++ /dev/null @@ -1,2 +0,0 @@ -Support underscore and comma as thousands separators in the fractional part -for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst deleted file mode 100644 index e07200f9a3fbbd..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-02-14-28-30.gh-issue-130662.EIgIR8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Accept leading zeros in precision and width fields for -:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, -3), '.016f')``. diff --git a/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst b/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst deleted file mode 100644 index d97d937376a9d0..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-02-14-36-28.gh-issue-130662.Gpr2GB.rst +++ /dev/null @@ -1,3 +0,0 @@ -+Accept leading zeros in precision and width fields for -+:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), -'.016f')``. diff --git a/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst b/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst deleted file mode 100644 index 1affb5e2aadab7..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-03-12-59-17.gh-issue-135069.xop30V.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix the "Invalid error handling" exception in -:class:`!encodings.idna.IncrementalDecoder` to correctly replace the -'errors' parameter. diff --git a/Misc/NEWS.d/next/Library/2025-06-10-21-42-04.gh-issue-135335.WnUqb_.rst b/Misc/NEWS.d/next/Library/2025-06-10-21-42-04.gh-issue-135335.WnUqb_.rst deleted file mode 100644 index 466ba0d232cd1f..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-10-21-42-04.gh-issue-135335.WnUqb_.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`multiprocessing`: Flush ``stdout`` and ``stderr`` after preloading -modules in the ``forkserver``. diff --git a/Misc/NEWS.d/next/Library/2025-06-14-12-06-55.gh-issue-135487.KdVFff.rst b/Misc/NEWS.d/next/Library/2025-06-14-12-06-55.gh-issue-135487.KdVFff.rst deleted file mode 100644 index 3ef51fa31dfac9..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-14-12-06-55.gh-issue-135487.KdVFff.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :meth:`!reprlib.Repr.repr_int` when given integers with more than -:func:`sys.get_int_max_str_digits` digits. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-16-15-03-03.gh-issue-135561.mJCN8D.rst b/Misc/NEWS.d/next/Library/2025-06-16-15-03-03.gh-issue-135561.mJCN8D.rst deleted file mode 100644 index ee743f161138e6..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-16-15-03-03.gh-issue-135561.mJCN8D.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a crash on DEBUG builds when an HACL* HMAC routine fails. Patch by -Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-17-22-44-19.gh-issue-119180.Ogv8Nj.rst b/Misc/NEWS.d/next/Library/2025-06-17-22-44-19.gh-issue-119180.Ogv8Nj.rst deleted file mode 100644 index c5e5d5b4f8d85e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-17-22-44-19.gh-issue-119180.Ogv8Nj.rst +++ /dev/null @@ -1,2 +0,0 @@ -Only fetch globals and locals if necessary in -:func:`annotationlib.get_annotations` diff --git a/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst b/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst deleted file mode 100644 index eabf5ea4aaa2ac..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-17-23-13-56.gh-issue-135557.Bfcy4v.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix races on :mod:`heapq` updates and :class:`list` reads on the :term:`free threaded ` -build. diff --git a/Misc/NEWS.d/next/Library/2025-06-18-11-43-17.gh-issue-135646.r7ekEn.rst b/Misc/NEWS.d/next/Library/2025-06-18-11-43-17.gh-issue-135646.r7ekEn.rst deleted file mode 100644 index 5fbd751467dd58..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-18-11-43-17.gh-issue-135646.r7ekEn.rst +++ /dev/null @@ -1 +0,0 @@ -Raise consistent :exc:`NameError` exceptions in :func:`annotationlib.ForwardRef.evaluate` diff --git a/Misc/NEWS.d/next/Library/2025-06-18-13-58-13.gh-issue-135645.109nff.rst b/Misc/NEWS.d/next/Library/2025-06-18-13-58-13.gh-issue-135645.109nff.rst deleted file mode 100644 index a7764a0105b8b3..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-18-13-58-13.gh-issue-135645.109nff.rst +++ /dev/null @@ -1,2 +0,0 @@ -Added ``supports_isolated_interpreters`` field to -:data:`sys.implementation`. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst b/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst deleted file mode 100644 index ad217b57b4b510..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-22-02-16-17.gh-issue-135640.FXyFL6.rst +++ /dev/null @@ -1,4 +0,0 @@ -Address bug where it was possible to call -:func:`xml.etree.ElementTree.ElementTree.write` on an ElementTree object with -an invalid root element. This behavior blanked the file passed to ``write`` -if it already existed. diff --git a/Misc/NEWS.d/next/Library/2025-06-22-16-23-44.gh-issue-135815.0DandH.rst b/Misc/NEWS.d/next/Library/2025-06-22-16-23-44.gh-issue-135815.0DandH.rst deleted file mode 100644 index 0f4a68bf745a8e..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-22-16-23-44.gh-issue-135815.0DandH.rst +++ /dev/null @@ -1,2 +0,0 @@ -:mod:`netrc`: skip security checks if :func:`os.getuid` is missing. -Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2025-06-23-10-19-11.gh-issue-135855.-J0AGF.rst b/Misc/NEWS.d/next/Library/2025-06-23-10-19-11.gh-issue-135855.-J0AGF.rst deleted file mode 100644 index fcf495bdceb168..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-23-10-19-11.gh-issue-135855.-J0AGF.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raise :exc:`TypeError` instead of :exc:`SystemError` when -:func:`!_interpreters.set___main___attrs` is passed a non-dict object. -Patch by Brian Schubert. diff --git a/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst b/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst deleted file mode 100644 index f93c9faee5863a..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-23-11-04-25.gh-issue-135836.-C-c4v.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could occur when the Happy Eyeballs algorithm resulted in an empty exceptions list during connection attempts. diff --git a/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst b/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst deleted file mode 100644 index 1d1e7a2298c085..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-24-10-52-35.gh-issue-135836.s37351.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could -occur when non-\ :exc:`OSError` exception is raised during connection and -socket's ``close()`` raises :exc:`!OSError`. diff --git a/Misc/NEWS.d/next/Library/2025-06-24-14-43-24.gh-issue-135878.Db4roX.rst b/Misc/NEWS.d/next/Library/2025-06-24-14-43-24.gh-issue-135878.Db4roX.rst deleted file mode 100644 index 969cf2dfa40878..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-24-14-43-24.gh-issue-135878.Db4roX.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixes a crash of :class:`types.SimpleNamespace` on :term:`free threading` builds, -when several threads were calling its :meth:`~object.__repr__` method at the -same time. diff --git a/Misc/NEWS.d/next/Library/2025-06-25-17-25-53.gh-issue-91555.xUpTLD.rst b/Misc/NEWS.d/next/Library/2025-06-25-17-25-53.gh-issue-91555.xUpTLD.rst deleted file mode 100644 index 4e5ccbc45abe01..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-25-17-25-53.gh-issue-91555.xUpTLD.rst +++ /dev/null @@ -1,4 +0,0 @@ -An earlier change, which was introduced in 3.14.0b2, has been reverted. It -disabled logging for a logger during handling of log messages for that -logger. Since the reversion, the behaviour should be as it was before -3.14.0b2. diff --git a/Misc/NEWS.d/next/Library/2025-06-26-11-52-40.gh-issue-53203.TMigBr.rst b/Misc/NEWS.d/next/Library/2025-06-26-11-52-40.gh-issue-53203.TMigBr.rst deleted file mode 100644 index ba2fae49fdc933..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-26-11-52-40.gh-issue-53203.TMigBr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :func:`time.strptime` for ``%c`` and ``%x`` formats on locales byn_ER, -wal_ET and lzh_TW, and for ``%X`` format on locales ar_SA, bg_BG and lzh_TW. diff --git a/Misc/NEWS.d/next/Library/2025-06-26-17-28-49.gh-issue-135995.pPrDCt.rst b/Misc/NEWS.d/next/Library/2025-06-26-17-28-49.gh-issue-135995.pPrDCt.rst deleted file mode 100644 index 998b3cd85b1d5d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-26-17-28-49.gh-issue-135995.pPrDCt.rst +++ /dev/null @@ -1 +0,0 @@ -In the palmos encoding, make byte ``0x9b`` decode to ``›`` (U+203A - SINGLE RIGHT-POINTING ANGLE QUOTATION MARK). diff --git a/Misc/NEWS.d/next/Library/2025-06-27-13-34-28.gh-issue-136028.RY727g.rst b/Misc/NEWS.d/next/Library/2025-06-27-13-34-28.gh-issue-136028.RY727g.rst deleted file mode 100644 index 9859df7cf6a69c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-27-13-34-28.gh-issue-136028.RY727g.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix parsing month names containing "İ" (U+0130, LATIN CAPITAL LETTER I WITH -DOT ABOVE) in :func:`time.strptime`. This affects locales az_AZ, ber_DZ, -ber_MA and crh_UA. diff --git a/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst b/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst deleted file mode 100644 index fc13eb1d9e0c99..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-30-11-12-24.gh-issue-85702.0Lrbwu.rst +++ /dev/null @@ -1,3 +0,0 @@ -If ``zoneinfo._common.load_tzdata`` is given a package without a resource a -:exc:`zoneinfo.ZoneInfoNotFoundError` is raised rather than a :exc:`PermissionError`. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst b/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst deleted file mode 100644 index dd5cecdf3a1209..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-05-06-56-16.gh-issue-136316.3zj_Do.rst +++ /dev/null @@ -1,2 +0,0 @@ -Improve support for evaluating nested forward references in -:func:`typing.evaluate_forward_ref`. diff --git a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst b/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst deleted file mode 100644 index 0a0d66ac0b8abf..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-05-09-45-04.gh-issue-136286.N67Amr.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix pickling failures for protocols 0 and 1 for many objects realted to -subinterpreters. diff --git a/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst b/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst deleted file mode 100644 index 71d15ee0852ebd..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-06-18-13-28-08.gh-issue-102555.nADrzJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix comment parsing in :class:`html.parser.HTMLParser` according to the -HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the -comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. diff --git a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst b/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst deleted file mode 100644 index b6f9e104e44047..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-06-25-14-13-39.gh-issue-135661.idjQ0B.rst +++ /dev/null @@ -1,25 +0,0 @@ -Fix parsing start and end tags in :class:`html.parser.HTMLParser` -according to the HTML5 standard. - -* Whitespaces no longer accepted between ```` does not end the script section. - -* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized - as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. - -* Null character (U+0000) no longer ends the tag name. - -* Attributes and slashes after the tag name in end tags are now ignored, - instead of terminating after the first ``>`` in quoted attribute value. - E.g. ````. - -* Multiple slashes and whitespaces between the last attribute and closing ``>`` - are now ignored in both start and end tags. E.g. ````. - -* Multiple ``=`` between attribute name and value are no longer collapsed. - E.g. ```` produces attribute "foo" with value "=bar". - -* Whitespaces between the ``=`` separator and attribute name or value are no - longer ignored. E.g. ```` produces two attributes "foo" and - "=bar", both with value None; ```` produces two attributes: - "foo" with value "" and "bar" with value None. diff --git a/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst b/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst deleted file mode 100644 index 93caed3aa3b9dd..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-06-27-21-23-19.gh-issue-136053.QZxcee.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`marshal`: fix a possible crash when deserializing :class:`slice` objects. diff --git a/Misc/NEWS.d/next/Tests/2025-06-19-15-29-38.gh-issue-135494.FVl9a0.rst b/Misc/NEWS.d/next/Tests/2025-06-19-15-29-38.gh-issue-135494.FVl9a0.rst deleted file mode 100644 index 832d1fe033e229..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-06-19-15-29-38.gh-issue-135494.FVl9a0.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix regrtest to support excluding tests from ``--pgo`` tests. Patch by -Victor Stinner. diff --git a/Misc/NEWS.d/next/Tests/2025-06-26-15-15-35.gh-issue-135966.EBpF8Y.rst b/Misc/NEWS.d/next/Tests/2025-06-26-15-15-35.gh-issue-135966.EBpF8Y.rst deleted file mode 100644 index 8dc007431f3919..00000000000000 --- a/Misc/NEWS.d/next/Tests/2025-06-26-15-15-35.gh-issue-135966.EBpF8Y.rst +++ /dev/null @@ -1 +0,0 @@ -The iOS testbed now handles the ``app_packages`` folder as a site directory. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-06-25-10-36-22.gh-issue-133600.bkdgHC.rst b/Misc/NEWS.d/next/Tools-Demos/2025-06-25-10-36-22.gh-issue-133600.bkdgHC.rst deleted file mode 100644 index 0225806768eb45..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-06-25-10-36-22.gh-issue-133600.bkdgHC.rst +++ /dev/null @@ -1,4 +0,0 @@ -Backport file reorganization for Tools/wasm/wasi. - -This should make backporting future code changes easier. It also simplifies -instructions around how to do WASI builds in the devguide. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-06-26-15-58-13.gh-issue-135968.C4v_-W.rst b/Misc/NEWS.d/next/Tools-Demos/2025-06-26-15-58-13.gh-issue-135968.C4v_-W.rst deleted file mode 100644 index 1c0b3825c71c1d..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-06-26-15-58-13.gh-issue-135968.C4v_-W.rst +++ /dev/null @@ -1 +0,0 @@ -Stubs for ``strip`` are now provided as part of an iOS install. diff --git a/README.rst b/README.rst index 59efafb69d925b..0092dfb497a021 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.14.0 beta 3 +This is Python version 3.14.0 beta 4 ==================================== .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push From f0e603412bb6945761551c412ad324720909c6cc Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 14:11:28 +0200 Subject: [PATCH 587/713] [3.14] gh-101100: Fix sphinx warnings in `Doc/library/exceptions.rst` (GH-136309) (#136414) Co-authored-by: Yuki Kobayashi Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/exceptions.rst | 30 +++++++++++++++++++----------- Doc/tools/.nitignore | 1 - 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index 9806ae80905ca0..8cc887b8ceb378 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -204,10 +204,16 @@ The following exceptions are the exceptions that are usually raised. assignment fails. (When an object does not support attribute references or attribute assignments at all, :exc:`TypeError` is raised.) - The :attr:`name` and :attr:`obj` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the attribute - that was attempted to be accessed and the object that was accessed for said - attribute, respectively. + The optional *name* and *obj* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the attribute that was attempted to be accessed. + + .. attribute:: obj + + The object that was accessed for the named attribute. .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. @@ -215,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and + without reading any data. (Note: the :meth:`!io.IOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -312,9 +318,11 @@ The following exceptions are the exceptions that are usually raised. unqualified names. The associated value is an error message that includes the name that could not be found. - The :attr:`name` attribute can be set using a keyword-only argument to the - constructor. When set it represent the name of the variable that was attempted - to be accessed. + The optional *name* keyword-only argument sets the attribute: + + .. attribute:: name + + The name of the variable that was attempted to be accessed. .. versionchanged:: 3.10 Added the :attr:`name` attribute. @@ -382,7 +390,7 @@ The following exceptions are the exceptions that are usually raised. The corresponding error message, as provided by the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` + functions :c:func:`!perror` under POSIX, and :c:func:`!FormatMessage` under Windows. .. attribute:: filename @@ -398,7 +406,7 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the + :exc:`!mmap.error` have been merged into :exc:`OSError`, and the constructor may return a subclass. .. versionchanged:: 3.4 @@ -590,7 +598,7 @@ The following exceptions are the exceptions that are usually raised. handled, the Python interpreter exits; no stack traceback is printed. The constructor accepts the same optional argument passed to :func:`sys.exit`. If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if + C's :c:func:`!exit` function); if it is ``None``, the exit status is zero; if it has another type (such as a string), the object's value is printed and the exit status is one. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index e29775b075a82f..4f5396857f3024 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -16,7 +16,6 @@ Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst Doc/library/email.parser.rst -Doc/library/exceptions.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst From 78c12273810036f555c515e807887b9d1473f12b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 14:48:03 +0200 Subject: [PATCH 588/713] [3.14] gh-136186: Fix race condition in test_external_inspection.test_only_active_thread (GH-136347) (#136416) --- Lib/test/test_external_inspection.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 0f31c225e68de3..44890ebfe5f76d 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -5,6 +5,7 @@ import sys import socket import threading +import time from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled @@ -930,9 +931,6 @@ def main_work(): # Signal threads to start waiting ready_event.set() - # Give threads time to start sleeping - time.sleep(0.1) - # Now do busy work to hold the GIL main_work() """ @@ -967,7 +965,23 @@ def main_work(): # Get stack trace with all threads unwinder_all = RemoteUnwinder(p.pid, all_threads=True) - all_traces = unwinder_all.get_stack_trace() + for _ in range(10): + # Wait for the main thread to start its busy work + all_traces = unwinder_all.get_stack_trace() + found = False + for thread_id, stack in all_traces: + if not stack: + continue + current_frame = stack[0] + if current_frame.funcname == "main_work" and current_frame.lineno >15: + found = True + + if found: + break + # Give a bit of time to take the next sample + time.sleep(0.1) + else: + self.fail("Main thread did not start its busy work on time") # Get stack trace with only GIL holder unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) From 72358614d2a4fba2430b0c2827841b1973fa710a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 8 Jul 2025 17:31:34 +0300 Subject: [PATCH 589/713] Post 3.14.0b4 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 40135791403f3c..77fc8aeb918cf6 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -24,7 +24,7 @@ #define PY_RELEASE_SERIAL 4 /* Version as a string */ -#define PY_VERSION "3.14.0b4" +#define PY_VERSION "3.14.0b4+" /*--end constants--*/ From f937b9fffbaaf5e604d5a575758c81d411b33c81 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 18:11:48 +0200 Subject: [PATCH 590/713] [3.14] gh-136380: Fix import behavior for `concurrent.futures.InterpreterPoolExecutor` (GH-136381) (#136420) gh-136380: Fix import behavior for `concurrent.futures.InterpreterPoolExecutor` (GH-136381) (cherry picked from commit 490eea02819ad303a5042529af7507b7b1fdabdc) Co-authored-by: AN Long Co-authored-by: Serhiy Storchaka Co-authored-by: Peter Bierma Co-authored-by: sobolevn --- Lib/concurrent/futures/__init__.py | 35 ++++++++-------- .../test_interpreter_pool.py | 41 +++++++++++++++++++ ...-07-07-22-12-32.gh-issue-136380.1b_nXl.rst | 3 ++ 3 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 7ada7431c1ab8c..e717222cf98b32 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,7 +17,7 @@ wait, as_completed) -__all__ = ( +__all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', @@ -29,10 +29,18 @@ 'Executor', 'wait', 'as_completed', - 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', -) +] + + +try: + import _interpreters +except ImportError: + _interpreters = None + +if _interpreters: + __all__.append('InterpreterPoolExecutor') def __dir__(): @@ -43,22 +51,15 @@ def __getattr__(name): global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor as pe - ProcessPoolExecutor = pe - return pe + from .process import ProcessPoolExecutor + return ProcessPoolExecutor if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor as te - ThreadPoolExecutor = te - return te + from .thread import ThreadPoolExecutor + return ThreadPoolExecutor - if name == 'InterpreterPoolExecutor': - try: - from .interpreter import InterpreterPoolExecutor as ie - except ModuleNotFoundError: - ie = InterpreterPoolExecutor = None - else: - InterpreterPoolExecutor = ie - return ie + if _interpreters and name == 'InterpreterPoolExecutor': + from .interpreter import InterpreterPoolExecutor + return InterpreterPoolExecutor raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index 844dfdd6fc901c..b10bbebd0984c4 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -2,7 +2,9 @@ import contextlib import io import os +import subprocess import sys +import textwrap import time import unittest from concurrent.futures.interpreter import BrokenInterpreterPool @@ -457,6 +459,45 @@ def test_free_reference(self): # Weak references don't cross between interpreters. raise unittest.SkipTest('not applicable') + @support.requires_subprocess() + def test_import_interpreter_pool_executor(self): + # Test the import behavior normally if _interpreters is unavailable. + code = textwrap.dedent(""" + import sys + # Set it to None to emulate the case when _interpreter is unavailable. + sys.modules['_interpreters'] = None + from concurrent import futures + + try: + futures.InterpreterPoolExecutor + except AttributeError: + pass + else: + print('AttributeError not raised!', file=sys.stderr) + sys.exit(1) + + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + pass + else: + print('ImportError not raised!', file=sys.stderr) + sys.exit(1) + + from concurrent.futures import * + + if 'InterpreterPoolExecutor' in globals(): + print('InterpreterPoolExecutor should not be imported!', + file=sys.stderr) + sys.exit(1) + """) + + cmd = [sys.executable, '-c', code] + p = subprocess.run(cmd, capture_output=True) + self.assertEqual(p.returncode, 0, p.stderr.decode()) + self.assertEqual(p.stdout.decode(), '') + self.assertEqual(p.stderr.decode(), '') + class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst b/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst new file mode 100644 index 00000000000000..4ac04b9c0a6c5c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst @@ -0,0 +1,3 @@ +Raises :exc:`AttributeError` when accessing +:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are +not available. From 66f85f4be84964d2ffb9611d05cebb986c2abf2d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 19:38:37 +0200 Subject: [PATCH 591/713] [3.14] Update bytecode magic number in tests for the 3.14 release candidate (GH-136427) (#136429) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/test/test_importlib/test_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 5de89714eb50c7..6d6d5f96aab4a8 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -635,7 +635,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3495 + EXPECTED_MAGIC_NUMBER = 3625 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( From 6a2a2906f805ce6b4e8315da32e7a4ad0aa6bace Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 8 Jul 2025 19:58:01 +0200 Subject: [PATCH 592/713] [3.14] GH-133136: Revise QSBR to reduce excess memory held (gh-135473) (#135912) The free threading build uses QSBR to delay the freeing of dictionary keys and list arrays when the objects are accessed by multiple threads in order to allow concurrent reads to proceed with holding the object lock. The requests are processed in batches to reduce execution overhead, but for large memory blocks this can lead to excess memory usage. Take into account the size of the memory block when deciding when to process QSBR requests. Also track the amount of memory being held by QSBR for mimalloc pages. Advance the write sequence if this memory exceeds a limit. Advancing the sequence will allow it to be freed more quickly. Process the held QSBR items from the "eval breaker", rather than from `_PyMem_FreeDelayed()`. This gives a higher chance that the global read sequence has advanced enough so that items can be freed. (cherry picked from commit 113de8545ffe74a4a1dddb9351fa1cbd3562b621) Co-authored-by: Neil Schemenauer Co-authored-by: Sam Gross --- Doc/data/python3.14.abi | 6772 +++++++++-------- Include/internal/pycore_pymem.h | 2 +- Include/internal/pycore_qsbr.h | 31 +- ...-06-03-21-06-22.gh-issue-133136.Usnvri.rst | 2 + Objects/codeobject.c | 2 +- Objects/dictobject.c | 4 +- Objects/listobject.c | 3 +- Objects/obmalloc.c | 96 +- Python/ceval_gil.c | 4 + Python/qsbr.c | 12 +- 10 files changed, 3518 insertions(+), 3410 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi index 478d23a2a58125..e2cb3cbe30da47 100644 --- a/Doc/data/python3.14.abi +++ b/Doc/data/python3.14.abi @@ -2166,7 +2166,7 @@ - + @@ -2671,15 +2671,15 @@ - + - + - + @@ -3954,7 +3954,7 @@ - + @@ -5069,15 +5069,12 @@ - - - - - - + + + - + @@ -5267,14 +5264,14 @@ - + - + @@ -6021,17 +6018,17 @@ - - + + - - + + - - - + + + @@ -6388,159 +6385,159 @@ - - + + - - + + - + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + - - - - + + + + - - - + + + - - - + + + - - - - - - - + + + + + + + - - - - + + + + - - - - + + + + - - + + - - + + - - + + - - - + + + - - - - - - - - + + + + + + + + - - - + + + - - - - + + + + - - - - - - + + + + + + - - - + + + - - + + - - + + - - + + @@ -7055,7 +7052,7 @@ - + @@ -7237,83 +7234,83 @@ - - - - - + + + + + - - - - + + + + - - + + - - - + + + - - - + + + - - - - + + + + - - - + + + - - - - + + + + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + - - - - - - + + + + + + @@ -8124,102 +8121,102 @@ - + - + - - + + - - + + - - - - + + + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + - - - + + + - - - + + + - - + + - - + + @@ -8414,8 +8411,8 @@ - - + + @@ -8593,54 +8590,54 @@ - - + + - - - + + + - - - - + + + + - - - - + + + + - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - + + + @@ -9023,27 +9020,27 @@ - - - + + + - - - - + + + + - - + + - - + + - - + + @@ -9182,62 +9179,62 @@ - - + + - - + + - - + + - - - + + + - - - - + + + + - - - + + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + @@ -9581,76 +9578,76 @@ - - - + + + - - + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - + + + - - + + - - + + - - + + - - + + - - + + - - + + @@ -10264,19 +10261,19 @@ - + - + - + - + @@ -10395,45 +10392,45 @@ - + - + - - - + + + - - - + + + - - + + - - + + - - + + - - - + + + - - + + - - + + @@ -10493,16 +10490,16 @@ - - + + - - + + - - + + @@ -10596,9 +10593,9 @@ - - - + + + @@ -10929,7 +10926,7 @@ - + @@ -10950,163 +10947,163 @@ - + - - + + - + - - + + - - + + - - - + + + - - - + + + - - + + - - - + + + - - + + - - + + - - + + - - - + + + - - - + + + - - - - + + + + - - + + - - + + - - - - - + + + + + - - - - + + + + - - + + - - + + - - + + - - - + + + - - + + - - + + - - - + + + - - - - + + + + - - - + + + - - + + - - + + - - - + + + - - - + + + - + - - + + - - - - - + + + + + @@ -13153,10 +13150,10 @@ - + - + @@ -14526,27 +14523,27 @@ - - - - - + + - - + + - + - + - + - - - + + + + + + @@ -14576,8 +14573,8 @@ - - + + @@ -14591,7 +14588,7 @@ - + @@ -14627,175 +14624,175 @@ - - + + - + - - + + - + - - + + - + - - + + - - + + - + - + - - + + - + - - + + - + - - + + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - - + + - + - + - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - - + + - + - - + + - + - + - + - - + + - + - - + + - + - + - + - + @@ -14806,8 +14803,8 @@ - - + + @@ -14821,7 +14818,7 @@ - + @@ -14929,14 +14926,14 @@ - + - - - + + + @@ -14944,14 +14941,14 @@ - - + + - - - + + + @@ -14963,7 +14960,7 @@ - + @@ -14975,7 +14972,7 @@ - + @@ -14984,10 +14981,10 @@ - - - - + + + + @@ -15043,8 +15040,8 @@ - - + + @@ -15052,9 +15049,9 @@ - - - + + + @@ -15062,7 +15059,7 @@ - + @@ -15070,8 +15067,8 @@ - - + + @@ -15103,8 +15100,8 @@ - - + + @@ -15169,16 +15166,16 @@ - + - + - + - + @@ -15265,7 +15262,7 @@ - + @@ -15313,43 +15310,43 @@ - - + + - + - + - + - - + + - + - + - - + + - + @@ -15358,7 +15355,7 @@ - + @@ -15379,188 +15376,188 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - - + + - + - + - + - - - + + + - + - + - + - + - - + + - + - - + + @@ -15574,49 +15571,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -15631,16 +15628,16 @@ - + - + - + @@ -15658,22 +15655,22 @@ - + - + - + - + - + @@ -15697,13 +15694,13 @@ - + - + @@ -15715,27 +15712,27 @@ - - + + - - - + + + - + - + - - + + @@ -15761,28 +15758,28 @@ - + - + - + - + - + - + - - - + + + @@ -15790,19 +15787,19 @@ - - + + - + - - + + - + @@ -15814,8 +15811,8 @@ - - + + @@ -15829,7 +15826,7 @@ - + @@ -15859,10 +15856,10 @@ - + - + @@ -15916,7 +15913,7 @@ - + @@ -15940,10 +15937,10 @@ - + - + @@ -15969,8 +15966,8 @@ - - + + @@ -15979,7 +15976,7 @@ - + @@ -15990,8 +15987,8 @@ - - + + @@ -16002,10 +15999,10 @@ - + - + @@ -16022,10 +16019,10 @@ - - + + - + @@ -16034,18 +16031,18 @@ - - - - + + + + - + - + @@ -16054,7 +16051,7 @@ - + @@ -16074,7 +16071,7 @@ - + @@ -16082,25 +16079,25 @@ - + - - - + + + - - + + - - + + @@ -16116,20 +16113,20 @@ - - + + - - - - - - + + + + + + @@ -16137,11 +16134,11 @@ - + - - + + @@ -16149,11 +16146,11 @@ - + - - + + @@ -16164,8 +16161,8 @@ - - + + @@ -16173,11 +16170,11 @@ - + - - + + @@ -16209,12 +16206,12 @@ - + - + - + @@ -16229,42 +16226,42 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -16276,49 +16273,49 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -16326,7 +16323,7 @@ - + @@ -16334,7 +16331,7 @@ - + @@ -16345,7 +16342,7 @@ - + @@ -16353,7 +16350,7 @@ - + @@ -16361,7 +16358,7 @@ - + @@ -16372,7 +16369,7 @@ - + @@ -16380,12 +16377,12 @@ - + - + @@ -16393,7 +16390,7 @@ - + @@ -16409,7 +16406,7 @@ - + @@ -16420,7 +16417,7 @@ - + @@ -16431,7 +16428,7 @@ - + @@ -16442,7 +16439,7 @@ - + @@ -16456,12 +16453,12 @@ - + - + @@ -16469,7 +16466,7 @@ - + @@ -16480,7 +16477,7 @@ - + @@ -16491,7 +16488,7 @@ - + @@ -16499,7 +16496,7 @@ - + @@ -16507,7 +16504,7 @@ - + @@ -16515,7 +16512,7 @@ - + @@ -16526,7 +16523,7 @@ - + @@ -16540,7 +16537,7 @@ - + @@ -16563,7 +16560,7 @@ - + @@ -16586,7 +16583,7 @@ - + @@ -16606,9 +16603,9 @@ - + - + @@ -17352,16 +17349,16 @@ - + - - - - - + + + + + @@ -17372,40 +17369,40 @@ - + - + - - - - + + + + - + - + - - - + + + - + - - + + - + - + - + @@ -17416,13 +17413,13 @@ - - + + - + - + @@ -17434,11 +17431,11 @@ - + - - + + @@ -17449,18 +17446,18 @@ - + - - + + - + - + - + @@ -17469,64 +17466,64 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -17537,7 +17534,7 @@ - + @@ -17587,7 +17584,7 @@ - + @@ -17616,7 +17613,7 @@ - + @@ -17642,7 +17639,7 @@ - + @@ -17677,7 +17674,7 @@ - + @@ -17685,7 +17682,7 @@ - + @@ -17699,7 +17696,7 @@ - + @@ -17710,7 +17707,7 @@ - + @@ -17721,7 +17718,7 @@ - + @@ -17735,7 +17732,7 @@ - + @@ -17746,7 +17743,7 @@ - + @@ -17754,7 +17751,7 @@ - + @@ -17765,7 +17762,7 @@ - + @@ -17776,7 +17773,7 @@ - + @@ -17790,7 +17787,7 @@ - + @@ -17798,7 +17795,7 @@ - + @@ -17812,7 +17809,7 @@ - + @@ -17820,7 +17817,7 @@ - + @@ -17840,8 +17837,8 @@ - - + + @@ -17849,12 +17846,12 @@ - + - + @@ -17863,8 +17860,8 @@ - - + + @@ -17881,21 +17878,21 @@ - + - + - + - + - + @@ -17904,7 +17901,7 @@ - + @@ -17924,7 +17921,7 @@ - + @@ -17964,7 +17961,7 @@ - + @@ -18026,7 +18023,7 @@ - + @@ -18040,24 +18037,24 @@ - + - + - + - + - + - + - + @@ -18066,2602 +18063,2602 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - - + + - + - + @@ -20670,22 +20667,22 @@ - - - - + + + + - - + + - + - - + + @@ -20693,10 +20690,10 @@ - + - + @@ -20714,57 +20711,57 @@ - + - + - - + + - + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + @@ -20773,7 +20770,7 @@ - + @@ -20799,19 +20796,19 @@ - + - - + + - + - + @@ -20823,20 +20820,20 @@ - + - + - + - + - + @@ -20845,10 +20842,10 @@ - + - + @@ -20857,7 +20854,7 @@ - + @@ -20865,17 +20862,17 @@ - + - + - + @@ -20886,7 +20883,7 @@ - + @@ -20894,19 +20891,19 @@ - + - - + + - + - + @@ -20914,10 +20911,10 @@ - - + + - + @@ -20926,7 +20923,7 @@ - + @@ -20937,7 +20934,7 @@ - + @@ -20951,16 +20948,16 @@ - + - + - + - + @@ -20984,21 +20981,21 @@ - + - + - + - + @@ -21006,7 +21003,7 @@ - + @@ -21029,13 +21026,13 @@ - + - + - + @@ -21046,12 +21043,12 @@ - + - + @@ -21065,7 +21062,7 @@ - + @@ -21079,7 +21076,7 @@ - + @@ -21088,7 +21085,7 @@ - + @@ -21105,8 +21102,8 @@ - - + + @@ -21123,10 +21120,10 @@ - + - + @@ -21134,13 +21131,13 @@ - + - + @@ -21156,7 +21153,7 @@ - + @@ -21164,15 +21161,15 @@ - + - + - + @@ -21185,7 +21182,7 @@ - + @@ -21196,12 +21193,12 @@ - + - + - + @@ -21224,19 +21221,19 @@ - - + + - + - + - + @@ -21245,15 +21242,15 @@ - + - + - + @@ -21261,10 +21258,10 @@ - + - + @@ -21275,7 +21272,7 @@ - + @@ -21284,7 +21281,7 @@ - + @@ -21292,10 +21289,10 @@ - + - + @@ -21307,8 +21304,8 @@ - - + + @@ -21316,12 +21313,12 @@ - + - + - + @@ -21335,7 +21332,7 @@ - + @@ -21343,18 +21340,18 @@ - + - + - + - + - + @@ -21368,8 +21365,8 @@ - - + + @@ -21380,7 +21377,7 @@ - + @@ -21404,28 +21401,28 @@ - + - + - + - + - + - + - + - + @@ -21458,7 +21455,7 @@ - + @@ -21470,7 +21467,7 @@ - + @@ -21479,19 +21476,19 @@ - + - + - + - + @@ -21506,7 +21503,7 @@ - + @@ -21518,10 +21515,10 @@ - + - + @@ -21533,25 +21530,25 @@ - + - + - + - + - + - + @@ -21572,40 +21569,40 @@ - + - + - + - + - + - + - + - + - + - + - + - + @@ -21626,13 +21623,13 @@ - + - + @@ -21653,22 +21650,22 @@ - + - + - + - + - + @@ -21676,7 +21673,7 @@ - + @@ -21729,21 +21726,21 @@ - + - + - + - + - - + + @@ -21754,14 +21751,14 @@ - - + + - - + + @@ -21769,71 +21766,71 @@ - + - + - + - - - + + + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -21842,17 +21839,17 @@ - + - + - + - + @@ -21876,7 +21873,7 @@ - + @@ -21884,26 +21881,26 @@ - - + + - + - + - - + + - + @@ -21912,7 +21909,7 @@ - + @@ -21920,86 +21917,95 @@ - + - + - + - + - - + + - + - + - + - + - + - + - + - - - - + - + + + + + + + + + + + + + - + - + - - + + - + - + - + - + - + - + - + - + @@ -22007,16 +22013,16 @@ - - + + - + - + @@ -22025,7 +22031,7 @@ - + @@ -22036,115 +22042,115 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22168,7 +22174,7 @@ - + @@ -22177,70 +22183,70 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -22249,34 +22255,34 @@ - + - + - + - + - + - + - + - + - + - + @@ -22290,7 +22296,7 @@ - + @@ -22298,25 +22304,25 @@ - + - + - + - + - + @@ -22339,32 +22345,32 @@ - + - + - + - - + + - + - + @@ -22372,19 +22378,19 @@ - - + + - - + + - + @@ -22393,8 +22399,8 @@ - - + + @@ -22405,14 +22411,14 @@ - + - + @@ -22429,7 +22435,7 @@ - + @@ -22437,9 +22443,9 @@ - + - + @@ -22448,15 +22454,15 @@ - + - + - + - + @@ -22471,7 +22477,7 @@ - + @@ -22483,13 +22489,13 @@ - + - + - + @@ -22500,9 +22506,9 @@ - + - + @@ -22523,28 +22529,28 @@ - + - + - - - + + + - + - + - - - + + + @@ -22558,23 +22564,23 @@ - + - + - + - + - + @@ -22585,7 +22591,7 @@ - + @@ -22593,43 +22599,43 @@ - - - - - - - - - - + + + + + + + + + + - + - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + - - + + @@ -22664,36 +22670,36 @@ - - - + + + - + - - - - - - - + + + + + + + - - + + - + - + - + - + @@ -22701,44 +22707,44 @@ - - + + - + - + - + - + - + - - + + - + - + - - + + - + @@ -22750,21 +22756,21 @@ - + - + - + - - - - - - - + + + + + + + @@ -22787,30 +22793,30 @@ - + - + - + - + - - + + - + - + - + - + @@ -22819,32 +22825,32 @@ - + - - - - - - + + + + + + - + - - - + + + - + - - + + @@ -22852,9 +22858,9 @@ - - - + + + @@ -22868,36 +22874,36 @@ - + - + - + - + - + - + - + - + - + - + - + @@ -22905,7 +22911,7 @@ - + @@ -22913,10 +22919,10 @@ - + - + @@ -22924,10 +22930,10 @@ - + - + @@ -22944,7 +22950,7 @@ - + @@ -22952,18 +22958,18 @@ - + - + - + - + - + @@ -22971,7 +22977,7 @@ - + @@ -22979,7 +22985,7 @@ - + @@ -22990,8 +22996,8 @@ - - + + @@ -23002,9 +23008,9 @@ - - - + + + @@ -23042,10 +23048,10 @@ - + - + @@ -23054,7 +23060,7 @@ - + @@ -23066,19 +23072,19 @@ - + - + - + - + @@ -23093,22 +23099,22 @@ - - + + - + - + - + @@ -23134,19 +23140,19 @@ - + - + - + - + @@ -23194,11 +23200,11 @@ - + - - + + @@ -23236,7 +23242,7 @@ - + @@ -23269,19 +23275,19 @@ - + - + - + - + @@ -23320,16 +23326,16 @@ - + - + - + @@ -23344,7 +23350,7 @@ - + @@ -23355,11 +23361,11 @@ - + - - + + @@ -23382,14 +23388,14 @@ - + - - + + @@ -23397,10 +23403,10 @@ - - + + - + @@ -23409,7 +23415,7 @@ - + @@ -23417,8 +23423,8 @@ - - + + @@ -23432,13 +23438,13 @@ - - + + - + @@ -23453,7 +23459,7 @@ - + @@ -23489,7 +23495,7 @@ - + @@ -23504,179 +23510,179 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - - + + + + + + + + + + + - - - - + + + - - + + - - - + + + + + + - - - - - - - + + + + - - + + - - - - - + + + + + - - + + - - - - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + - - + + - - - - - - - - - - - + + + + + + + + + + + - - - - - - - - - - + + + + + + + + + + - + - - - + - - - - - - - - - - + + + + + + + + + + + + - - - + + + - - - + + + - - - + + + + - - + - - - - - - + + + + + + @@ -23977,272 +23983,272 @@ - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - + - - + + - - + + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -24357,14 +24363,14 @@ - - + + - + - + @@ -24382,9 +24388,9 @@ - + - + @@ -24529,9 +24535,9 @@ - - - + + + @@ -24539,7 +24545,7 @@ - + @@ -24547,12 +24553,12 @@ - + - + @@ -24566,9 +24572,9 @@ - - - + + + @@ -24577,8 +24583,8 @@ + - @@ -24635,7 +24641,7 @@ - + @@ -24723,7 +24729,7 @@ - + @@ -24773,7 +24779,7 @@ - + @@ -24784,8 +24790,8 @@ - - + + @@ -24793,10 +24799,10 @@ - + - + @@ -24805,13 +24811,13 @@ - - + + - + @@ -24832,11 +24838,11 @@ - + - - + + @@ -24850,17 +24856,17 @@ - - - - - + + + + + - + - + @@ -24869,7 +24875,7 @@ - + @@ -24882,7 +24888,7 @@ - + @@ -24896,24 +24902,24 @@ - - + + - + - + - + @@ -24937,7 +24943,7 @@ - + @@ -25018,7 +25024,7 @@ - + @@ -25026,7 +25032,7 @@ - + @@ -25119,35 +25125,35 @@ - - + + - + - + - - + + - + - + - + - - + + - + - - + + @@ -25158,27 +25164,27 @@ - + - + - + - - - - - - - - - - - + + + + + + + + + + + @@ -25227,13 +25233,13 @@ - + - + @@ -25241,10 +25247,10 @@ - - - - + + + + @@ -25456,15 +25462,15 @@ - + - + - + @@ -25490,23 +25496,23 @@ - + - + - - + + - + @@ -25600,22 +25606,22 @@ - + - + - + - + @@ -25730,24 +25736,24 @@ - + - + - - - - - + + + + + - - - - + + + + @@ -25765,26 +25771,26 @@ - + - + - + - + - + - + @@ -25792,38 +25798,38 @@ - - + + - + - + - + - + - + - - + + - - + + @@ -25848,7 +25854,7 @@ - + @@ -25860,7 +25866,7 @@ - + @@ -25892,7 +25898,7 @@ - + @@ -25969,10 +25975,10 @@ - - + + - + @@ -25980,7 +25986,7 @@ - + @@ -25995,7 +26001,7 @@ - + @@ -26004,22 +26010,22 @@ - - + + - + - - + + - - + + - + - + @@ -26057,16 +26063,16 @@ - - + + - - + + @@ -26082,25 +26088,25 @@ - + - + - + - + - + - - + + @@ -26111,17 +26117,17 @@ - + - - + + - - + + @@ -26132,17 +26138,17 @@ - + - + - + - + @@ -26151,15 +26157,15 @@ - + - + - + @@ -26169,8 +26175,8 @@ - - + + @@ -26178,25 +26184,25 @@ - - + + - + - - + + - + - + @@ -26214,10 +26220,10 @@ - + - + @@ -26243,7 +26249,7 @@ - + @@ -26261,7 +26267,7 @@ - + @@ -26306,199 +26312,199 @@ - + - + - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - - - - + + + + + - - - + + + - - + + - + - + - + - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - + - + - - + + - - - - + + + + - - - + + + - + - + - + - + @@ -26508,15 +26514,15 @@ - - + + - - + + @@ -26526,71 +26532,71 @@ - + - + - + - - + + - + - + - - - - - - + + + + + + - + - + - + - + - + - - + + - - - - + + + + - + @@ -26611,10 +26617,10 @@ - + - + @@ -26626,24 +26632,24 @@ - + - + - - - - - + + + + + - + @@ -26651,33 +26657,33 @@ - + - + - + - - + + - + - + @@ -26686,24 +26692,24 @@ - - + + - - + + - + - + - + @@ -26713,17 +26719,17 @@ - - + + - - + + - + @@ -26738,7 +26744,7 @@ - + @@ -26747,14 +26753,14 @@ - + - + @@ -26766,11 +26772,11 @@ - + - + @@ -26784,50 +26790,50 @@ - + - + - + - + - + - + - + - - + + - + - + - + - + @@ -26868,21 +26874,21 @@ - - - - + + + + + - - - + + - + - + @@ -26897,27 +26903,27 @@ - + - + - + - + - + @@ -26940,9 +26946,9 @@ - - - + + + @@ -26950,10 +26956,10 @@ - - + + - + @@ -26962,7 +26968,7 @@ - + @@ -26976,8 +26982,8 @@ - - + + @@ -26990,10 +26996,10 @@ - - - - + + + + @@ -27001,13 +27007,13 @@ - + - - + + - + @@ -27016,7 +27022,7 @@ - + @@ -27042,23 +27048,23 @@ - - - - - + + + + + + - - - + + - - + + @@ -27071,10 +27077,10 @@ - + - + @@ -27115,18 +27121,18 @@ - + - + - + - + @@ -27182,31 +27188,31 @@ - + - + - + - + - + - + - + - + @@ -27217,151 +27223,151 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - + + - - + + - - + + - + - + - + - + - - - + + + - - + + - + - + - + - + - + - - + + - + @@ -27381,18 +27387,18 @@ - + - + - + @@ -27402,49 +27408,49 @@ - + - + - + - + - - - + + + - + - + - + - + @@ -27582,27 +27588,27 @@ - + - + - + - + - - - - - + + + + + @@ -27646,7 +27652,7 @@ - + @@ -27703,21 +27709,21 @@ - - - - + + + + - - - - + + + + - + @@ -27791,9 +27797,9 @@ - + - + @@ -27833,17 +27839,17 @@ - + - + - - - - + + + + @@ -27864,7 +27870,7 @@ - + @@ -27878,7 +27884,7 @@ - + @@ -27886,18 +27892,18 @@ - - - - - - - - - + + + + + + + + + - + @@ -27954,7 +27960,7 @@ - + @@ -27989,15 +27995,15 @@ - + - + - + @@ -28048,7 +28054,7 @@ - + @@ -28056,7 +28062,7 @@ - + @@ -28064,7 +28070,7 @@ - + @@ -28087,8 +28093,8 @@ - - + + @@ -28100,11 +28106,11 @@ - - + + - + @@ -28122,7 +28128,7 @@ - + @@ -28135,7 +28141,7 @@ - + @@ -28144,22 +28150,22 @@ - + - - + + - - + + @@ -28176,7 +28182,7 @@ - + @@ -28185,7 +28191,7 @@ - + @@ -28193,24 +28199,24 @@ - + - + - + - + - + - + @@ -28220,9 +28226,9 @@ - - - + + + @@ -28230,12 +28236,12 @@ - + + - - - + + @@ -28247,37 +28253,37 @@ - + - + - + - + - + - + - + - + + - @@ -28308,7 +28314,7 @@ - + @@ -28367,7 +28373,7 @@ - + @@ -28432,7 +28438,7 @@ - + @@ -28444,16 +28450,16 @@ - + - - + + - + - + @@ -28468,16 +28474,16 @@ - - + + - + - + - + @@ -28489,27 +28495,27 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + - - + + - - + + - + @@ -28577,13 +28583,13 @@ - + - - + + @@ -28608,21 +28614,21 @@ - + - + - + - + @@ -28631,7 +28637,7 @@ - + @@ -28645,7 +28651,7 @@ - + @@ -28654,7 +28660,7 @@ - + @@ -28722,12 +28728,12 @@ - + - + @@ -28744,19 +28750,19 @@ - + - + - + @@ -28791,12 +28797,12 @@ - + - + @@ -28809,44 +28815,44 @@ - + - + - + - + - + - + - + - + - + @@ -28855,32 +28861,32 @@ - + - + - + - + - + @@ -28905,7 +28911,7 @@ - + @@ -28913,12 +28919,12 @@ - - + + - + @@ -28928,33 +28934,33 @@ - + - + - + - + - + @@ -28962,85 +28968,85 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29049,62 +29055,62 @@ - + - + - + - + - - + + - + - + - + - - - + + + - + - - + + - + - - - - - + + + + + @@ -29116,8 +29122,8 @@ - - + + @@ -29163,12 +29169,12 @@ - + - + @@ -29176,8 +29182,8 @@ - - + + @@ -29186,53 +29192,53 @@ - + - + - + - + - + - - - + + + - + - - + + - + - - - - - - + + + + + + @@ -29248,7 +29254,7 @@ - + @@ -29257,21 +29263,21 @@ - + - + - + - + @@ -29287,65 +29293,65 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -29353,18 +29359,18 @@ - - + + - + - + - + @@ -29382,13 +29388,13 @@ - + - + - + @@ -29400,16 +29406,16 @@ - + - + - - - - + + + + @@ -29434,23 +29440,23 @@ - - + + - - + + - - + + - - + + - + @@ -29511,21 +29517,21 @@ - + - - - - + + + + - + - + @@ -29534,27 +29540,27 @@ - + - + - + - + - + @@ -29620,16 +29626,16 @@ - - + + - + - + @@ -29643,7 +29649,7 @@ - + @@ -29655,23 +29661,23 @@ - - + + - + - + - + @@ -29689,11 +29695,11 @@ - + - + @@ -29703,66 +29709,66 @@ - + - + - + - + - - - - + + + + - + - - + + - + - + - + - + - - + + - + - + - + - + - + - + @@ -29770,7 +29776,7 @@ - + @@ -29778,59 +29784,59 @@ - + - + - + - + - - - - - + + + + + - + - + - + - + - + - - - + + + - - - - - - - + + + + + + + @@ -30000,13 +30006,13 @@ - + - - + + @@ -30038,7 +30044,7 @@ - + @@ -30244,41 +30250,41 @@ - + - + - + - + - + - + - + - + - + @@ -30386,7 +30392,7 @@ - + @@ -30395,8 +30401,8 @@ - - + + @@ -30431,7 +30437,7 @@ - + @@ -30461,19 +30467,19 @@ - + - - + + - - - - + + + + @@ -30493,7 +30499,7 @@ - + @@ -30553,20 +30559,20 @@ - + - + - + - + @@ -30579,24 +30585,24 @@ - + - + - + - + - + @@ -30677,11 +30683,11 @@ - + - + @@ -30702,7 +30708,7 @@ - + @@ -30896,7 +30902,7 @@ - + @@ -30910,7 +30916,7 @@ - + @@ -30947,22 +30953,22 @@ - - - - + + + + - + + - + - @@ -31093,7 +31099,7 @@ - + @@ -31102,8 +31108,8 @@ - - + + @@ -31127,7 +31133,7 @@ - + @@ -31219,13 +31225,13 @@ - + - - - - + + + + @@ -31255,7 +31261,7 @@ - + @@ -31276,8 +31282,8 @@ - - + + @@ -31299,26 +31305,26 @@ - - - - - - - - - - - - + + + + + + + + + + + + - - - + + + @@ -31336,30 +31342,30 @@ - + - + - + - - + + - + - + @@ -31372,7 +31378,7 @@ - + @@ -31402,7 +31408,7 @@ - + @@ -31411,17 +31417,17 @@ - + - + - - + + @@ -31430,11 +31436,11 @@ - + - + @@ -31469,13 +31475,13 @@ - + - + @@ -31489,8 +31495,8 @@ - - + + @@ -31498,7 +31504,7 @@ - + diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 02537bdfef8598..3e12084b82ab26 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -88,7 +88,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr); +extern void _PyMem_FreeDelayed(void *ptr, size_t size); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index b835c3abaf5d0b..1f9b3fcf777493 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -48,8 +48,21 @@ struct _qsbr_thread_state { // Thread state (or NULL) PyThreadState *tstate; - // Used to defer advancing write sequence a fixed number of times - int deferrals; + // Number of held items added by this thread since the last write sequence + // advance + int deferred_count; + + // Estimate for the amount of memory that is held by this thread since + // the last write sequence advance + size_t deferred_memory; + + // Amount of memory in mimalloc pages deferred from collection. When + // deferred, they are prevented from being used for a different size class + // and in a different thread. + size_t deferred_page_memory; + + // True if the deferred memory frees should be processed. + bool should_process; // Is this thread state allocated? bool allocated; @@ -109,11 +122,17 @@ _Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) extern uint64_t _Py_qsbr_advance(struct _qsbr_shared *shared); -// Batches requests to advance the write sequence. This advances the write -// sequence every N calls, which reduces overhead but increases time to -// reclamation. Returns the new goal. +// Return the next value for the write sequence (current plus the increment). extern uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); +_Py_qsbr_shared_next(struct _qsbr_shared *shared); + +// Return true if deferred memory frees held by QSBR should be processed to +// determine if they can be safely freed. +static inline bool +_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr) +{ + return qsbr->should_process; +} // Have the read sequences advanced to the given goal? If this returns true, // it safe to reclaim any memory tagged with the goal (or earlier goal). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst new file mode 100644 index 00000000000000..a9501c13c95b3a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst @@ -0,0 +1,2 @@ +Limit excess memory usage in the :term:`free threading` build when a +large dictionary or list is resized and accessed by multiple threads. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 366622c95ec881..ba178abc0c071e 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -3368,7 +3368,7 @@ create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx) } memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *)); _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc); - _PyMem_FreeDelayed(tlbc); + _PyMem_FreeDelayed(tlbc, tlbc->size * sizeof(void *)); tlbc = new_tlbc; } char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co)); diff --git a/Objects/dictobject.c b/Objects/dictobject.c index b4940315a6c28d..be62ae5eefd00d 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -813,7 +813,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(keys); + _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys)); return; } #endif @@ -858,7 +858,7 @@ free_values(PyDictValues *values, bool use_qsbr) assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(values); + _PyMem_FreeDelayed(values, values_size_from_count(values->capacity)); return; } #endif diff --git a/Objects/listobject.c b/Objects/listobject.c index c5895645a2dd12..23d3472b6d4153 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -61,7 +61,8 @@ free_list_items(PyObject** items, bool use_qsbr) #ifdef Py_GIL_DISABLED _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); if (use_qsbr) { - _PyMem_FreeDelayed(array); + size_t size = sizeof(_PyListArray) + array->allocated * sizeof(PyObject *); + _PyMem_FreeDelayed(array, size); } else { PyMem_Free(array); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index d3931aab623b70..092be84d2b9954 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -124,6 +124,33 @@ _PyMem_mi_page_is_safe_to_free(mi_page_t *page) } +#ifdef Py_GIL_DISABLED + +// If we are deferring collection of more than this amount of memory for +// mimalloc pages, advance the write sequence. Advancing allows these +// pages to be re-used in a different thread or for a different size class. +#define QSBR_PAGE_MEM_LIMIT 4096*20 + +// Return true if the global write sequence should be advanced for a mimalloc +// page that is deferred from collection. +static bool +should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page) +{ + size_t bsize = mi_page_block_size(page); + size_t page_size = page->capacity*bsize; + if (page_size > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + qsbr->deferred_page_memory += page_size; + if (qsbr->deferred_page_memory > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + return false; +} +#endif + static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) { @@ -139,7 +166,14 @@ _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) _PyMem_mi_page_clear_qsbr(page); page->retire_expire = 0; - page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + + if (should_advance_qsbr_for_page(tstate->qsbr, page)) { + page->qsbr_goal = _Py_qsbr_advance(tstate->qsbr->shared); + } + else { + page->qsbr_goal = _Py_qsbr_shared_next(tstate->qsbr->shared); + } + llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); return false; } @@ -1141,8 +1175,44 @@ free_work_item(uintptr_t ptr, delayed_dealloc_cb cb, void *state) } } + +#ifdef Py_GIL_DISABLED + +// For deferred advance on free: the number of deferred items before advancing +// the write sequence. This is based on WORK_ITEMS_PER_CHUNK. We ideally +// want to process a chunk before it overflows. +#define QSBR_DEFERRED_LIMIT 127 + +// If the deferred memory exceeds 1 MiB, advance the write sequence. This +// helps limit memory usage due to QSBR delaying frees too long. +#define QSBR_FREE_MEM_LIMIT 1024*1024 + +// Return true if the global write sequence should be advanced for a deferred +// memory free. +static bool +should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) +{ + if (size > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + qsbr->deferred_count++; + qsbr->deferred_memory += size; + if (qsbr->deferred_count > QSBR_DEFERRED_LIMIT || + qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + return false; +} +#endif + static void -free_delayed(uintptr_t ptr) +free_delayed(uintptr_t ptr, size_t size) { #ifndef Py_GIL_DISABLED free_work_item(ptr, NULL, NULL); @@ -1200,23 +1270,32 @@ free_delayed(uintptr_t ptr) } assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); - uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + uint64_t seq; + if (should_advance_qsbr_for_free(tstate->qsbr, size)) { + seq = _Py_qsbr_advance(tstate->qsbr->shared); + } + else { + seq = _Py_qsbr_shared_next(tstate->qsbr->shared); + } buf->array[buf->wr_idx].ptr = ptr; buf->array[buf->wr_idx].qsbr_goal = seq; buf->wr_idx++; if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // Normally the processing of delayed items is done from the eval + // breaker. Processing here is a safety measure to ensure too much + // work does not accumulate. _PyMem_ProcessDelayed((PyThreadState *)tstate); } #endif } void -_PyMem_FreeDelayed(void *ptr) +_PyMem_FreeDelayed(void *ptr, size_t size) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed((uintptr_t)ptr); + free_delayed((uintptr_t)ptr, size); } } @@ -1226,7 +1305,10 @@ _PyObject_XDecRefDelayed(PyObject *ptr) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed(((uintptr_t)ptr)|0x01); + // We use 0 as the size since we don't have an easy way to know the + // actual size. If we are freeing many objects, the write sequence + // will be advanced due to QSBR_DEFERRED_LIMIT. + free_delayed(((uintptr_t)ptr)|0x01, 0); } } #endif @@ -1302,6 +1384,8 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate_impl->qsbr->should_process = false; + // Process thread-local work process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 57d8f68b000b60..aa68371ac8febf 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1387,6 +1387,10 @@ _Py_HandlePending(PyThreadState *tstate) _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); _Py_brc_merge_refcounts(tstate); } + /* Process deferred memory frees held by QSBR */ + if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + _PyMem_ProcessDelayed(tstate); + } #endif /* GC scheduled to run */ diff --git a/Python/qsbr.c b/Python/qsbr.c index bf34fb2523dfc8..c9fad5c05ef108 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -41,10 +41,6 @@ // Starting size of the array of qsbr thread states #define MIN_ARRAY_SIZE 8 -// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing -// the write sequence. -#define QSBR_DEFERRED_LIMIT 10 - // Allocate a QSBR thread state from the freelist static struct _qsbr_thread_state * qsbr_allocate(struct _qsbr_shared *shared) @@ -117,13 +113,9 @@ _Py_qsbr_advance(struct _qsbr_shared *shared) } uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +_Py_qsbr_shared_next(struct _qsbr_shared *shared) { - if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { - return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; - } - qsbr->deferrals = 0; - return _Py_qsbr_advance(qsbr->shared); + return _Py_qsbr_shared_current(shared) + QSBR_INCR; } static uint64_t From 9a79c5128f6d10118b9c2a402f2f9582f7a2c489 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 03:21:56 +0200 Subject: [PATCH 593/713] [3.14] gh-91048: Revert the memory cache removal for remote debugging (GH-136440) (#136443) gh-91048: Revert the memory cache removal for remote debugging (GH-136440) (cherry picked from commit 77d25e5b169f7c306d3a6d9ca6777c0a0be80d8f) gh-91048: Reintroduce the memory cache for remote debugging Co-authored-by: Pablo Galindo Salgado --- Modules/_remote_debugging_module.c | 10 ++++ Python/remote_debug.h | 78 ++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index e744fe65815985..7b8d272e63b160 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -945,6 +945,10 @@ parse_coro_chain( return -1; } + if (name == NULL) { + return 0; + } + if (PyList_Append(render_to, name)) { Py_DECREF(name); set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); @@ -2762,6 +2766,7 @@ _remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self } exit: + _Py_RemoteDebug_ClearCache(&self->handle); return result; } @@ -2885,9 +2890,11 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s goto result_err; } + _Py_RemoteDebug_ClearCache(&self->handle); return result; result_err: + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2954,9 +2961,11 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject goto cleanup; } + _Py_RemoteDebug_ClearCache(&self->handle); return result; cleanup: + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; } @@ -2982,6 +2991,7 @@ RemoteUnwinder_dealloc(PyObject *op) } #endif if (self->handle.pid != 0) { + _Py_RemoteDebug_ClearCache(&self->handle); _Py_RemoteDebug_CleanupProcHandle(&self->handle); } PyObject_Del(self); diff --git a/Python/remote_debug.h b/Python/remote_debug.h index d1fcb478d2b035..8f9b6cd4c4960f 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -110,6 +110,14 @@ get_page_size(void) { return page_size; } +typedef struct page_cache_entry { + uintptr_t page_addr; // page-aligned base address + char *data; + int valid; + struct page_cache_entry *next; +} page_cache_entry_t; + +#define MAX_PAGES 1024 // Define a platform-independent process handle structure typedef struct { @@ -121,9 +129,27 @@ typedef struct { #elif defined(__linux__) int memfd; #endif + page_cache_entry_t pages[MAX_PAGES]; Py_ssize_t page_size; } proc_handle_t; +static void +_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + PyMem_RawFree(handle->pages[i].data); + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } +} + +UNUSED static void +_Py_RemoteDebug_ClearCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].valid = 0; + } +} #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static mach_port_t pid_to_task(pid_t pid); @@ -152,6 +178,10 @@ _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->memfd = -1; #endif handle->page_size = get_page_size(); + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } return 0; } @@ -170,6 +200,7 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { } #endif handle->pid = 0; + _Py_RemoteDebug_FreePageCache(handle); } #if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX @@ -1035,6 +1066,53 @@ _Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, size_t size, void *out) { + size_t page_size = handle->page_size; + uintptr_t page_base = addr & ~(page_size - 1); + size_t offset_in_page = addr - page_base; + + if (offset_in_page + size > page_size) { + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); + } + + // Search for valid cached page + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (entry->valid && entry->page_addr == page_base) { + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + + // Find reusable slot + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (!entry->valid) { + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); + if (entry->data == NULL) { + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; + } + } + + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact ammount as a fallback + PyErr_Clear(); + goto fallback; + } + + entry->page_addr = page_base; + entry->valid = 1; + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + +fallback: + // Cache full — fallback to uncached read return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); } From e0b9f0721c842a32fb5fd761f2d3e5cd0a568a69 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 04:42:49 +0200 Subject: [PATCH 594/713] [3.14] gh-136229: Remove Platform Emscripten is not supported warning (GH-136230) (#136445) Updates configure script to identify Emscripten as Tier 3. (cherry picked from commit 6ea425828540d7a19296183c3410283897767d9a) Co-authored-by: Hood Chatham --- configure | 2 ++ configure.ac | 43 ++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/configure b/configure index 1646efe5f061ca..d32ea2dabc2be7 100755 --- a/configure +++ b/configure @@ -7254,6 +7254,8 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-linux-android/clang) : PY_SUPPORT_TIER=3 ;; #( + wasm32-*-emscripten/emcc) : + PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 diff --git a/configure.ac b/configure.ac index d868f7b26bad1c..ab688d93990a2d 100644 --- a/configure.ac +++ b/configure.ac @@ -1208,27 +1208,28 @@ dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. dnl AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], - [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc - [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version - [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC - [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - - [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang - [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], - [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang - [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang - - [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC - [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc - [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang - [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 - [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 - [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 - [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 - [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc + [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version + [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC + [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC + + [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang + [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], + [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang + [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang + + [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC + [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc + [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang + [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc + [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [wasm32-*-emscripten/emcc], [PY_SUPPORT_TIER=3], dnl Emscripten [PY_SUPPORT_TIER=0] ) From 2cb579283630fca4f831622b264158b10294e4d7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:00:30 +0200 Subject: [PATCH 595/713] [3.14] gh-94503: Update logging cookbook example with info on addressing log injection. (GH-136446) (GH-136449) Co-authored-by: Vinay Sajip Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/howto/logging-cookbook.rst | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index ae2697fbce30ad..52537a91df542c 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -4140,6 +4140,42 @@ The script, when run, prints something like: 2025-07-02 13:54:47,234 DEBUG fool me ... 2025-07-02 13:54:47,234 DEBUG can't get fooled again +If, on the other hand, you are concerned about `log injection +`_, you can use a +formatter which escapes newlines, as per the following example: + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + + class EscapingFormatter(logging.Formatter): + def format(self, record): + s = super().format(record) + return s.replace('\n', r'\n') + + if __name__ == '__main__': + h = logging.StreamHandler() + h.setFormatter(EscapingFormatter('%(asctime)s %(levelname)-9s %(message)s')) + logging.basicConfig(level=logging.DEBUG, handlers = [h]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +You can, of course, use whatever escaping scheme makes the most sense for you. +The script, when run, should produce output like this: + +.. code-block:: text + + 2025-07-09 06:47:33,783 DEBUG Single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me once ... + 2025-07-09 06:47:33,783 DEBUG Another single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me ...\ncan't get fooled again + +Escaping behaviour can't be the stdlib default , as it would break backwards +compatibility. .. patterns-to-avoid: From 85df16b7fa6db7dc0a389ecff9119f72716cee02 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:07:18 +0200 Subject: [PATCH 596/713] [3.14] gh-53243: Document `codecs.readbuffer_encode()` (GH-136284) (#136452) gh-53243: Document `codecs.readbuffer_encode()` (GH-136284) Closes GH-53243 (cherry picked from commit f1dcf3c7bf90961b8d5475154d3f28cfef0a054f) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/codecs.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index b231fa568cf342..a527d6746b5d07 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -265,6 +265,20 @@ wider range of codecs when working with binary files: :func:`iterencode`. +.. function:: readbuffer_encode(buffer, errors=None, /) + + Return a :class:`tuple` containing the raw bytes of *buffer*, a + :ref:`buffer-compatible object ` or :class:`str` + (encoded to UTF-8 before processing), and their length in bytes. + + The *errors* argument is ignored. + + .. code-block:: pycon + + >>> codecs.readbuffer_encode(b"Zito") + (b'Zito', 4) + + The module also provides the following constants which are useful for reading and writing to platform dependent files: From a9d2f08b579a400e0f2a337116bb29f476249dff Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:07:34 +0200 Subject: [PATCH 597/713] [3.14] gh-136162: Document `encodings` package functions (GH-136164) (#136454) gh-136162: Document `encodings` package functions (GH-136164) Closes GH-136162. (cherry picked from commit ffd7f2f231f5543e6863c6c85e86f72233229771) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/codecs.rst | 60 ++++++++++++++++++++++++++++++++++++++++++ Doc/whatsnew/3.9.rst | 2 +- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index a527d6746b5d07..0e84f18dd4d5d5 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1498,6 +1498,66 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. +:mod:`encodings` --- Encodings package +-------------------------------------- + +.. module:: encodings + :synopsis: Encodings package + +This module implements the following functions: + +.. function:: normalize_encoding(encoding) + + Normalize encoding name *encoding*. + + Normalization works as follows: all non-alphanumeric characters except the + dot used for Python package names are collapsed and replaced with a single + underscore, leading and trailing underscores are removed. + For example, ``' -;#'`` becomes ``'_'``. + + Note that *encoding* should be ASCII only. + + +.. note:: + The following functions should not be used directly, except for testing + purposes; :func:`codecs.lookup` should be used instead. + + +.. function:: search_function(encoding) + + Search for the codec module corresponding to the given encoding name + *encoding*. + + This function first normalizes the *encoding* using + :func:`normalize_encoding`, then looks for a corresponding alias. + It attempts to import a codec module from the encodings package using either + the alias or the normalized name. If the module is found and defines a valid + ``getregentry()`` function that returns a :class:`codecs.CodecInfo` object, + the codec is cached and returned. + + If the codec module defines a ``getaliases()`` function any returned aliases + are registered for future use. + + +.. function:: win32_code_page_search_function(encoding) + + Search for a Windows code page encoding *encoding* of the form ``cpXXXX``. + + If the code page is valid and supported, return a :class:`codecs.CodecInfo` + object for it. + + .. availability:: Windows. + + .. versionadded:: 3.14 + + +This module implements the following exception: + +.. exception:: CodecRegistryError + + Raised when a codec is invalid or incompatible. + + :mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------ diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 7fd9e6ac66e6c8..40d4a27bff9fee 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -1139,7 +1139,7 @@ Changes in the Python API (Contributed by Christian Heimes in :issue:`36384`). * :func:`codecs.lookup` now normalizes the encoding name the same way as - :func:`!encodings.normalize_encoding`, except that :func:`codecs.lookup` also + :func:`encodings.normalize_encoding`, except that :func:`codecs.lookup` also converts the name to lower case. For example, ``"latex+latin1"`` encoding name is now normalized to ``"latex_latin1"``. (Contributed by Jordon Xu in :issue:`37751`.) From c29fce05f3b4558f2dcd91ccaac5ab646caf6c1c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 10:34:19 +0200 Subject: [PATCH 598/713] [3.14] gh-134657: Remove newly added private names from asyncio.__all__ (GH-134665) (#136455) gh-134657: Remove newly added private names from asyncio.__all__ (GH-134665) (cherry picked from commit 797abd1f7fdeb744bf9f683ef844e7279aad3d72) Co-authored-by: Jelle Zijlstra --- Lib/asyncio/__init__.py | 29 ++++++++++------ Lib/asyncio/events.py | 8 ++--- Lib/asyncio/unix_events.py | 1 - Lib/test/libregrtest/save_env.py | 2 +- Lib/test/test_asyncgen.py | 2 +- Lib/test/test_asyncio/test_base_events.py | 2 +- Lib/test/test_asyncio/test_buffered_proto.py | 2 +- Lib/test/test_asyncio/test_context.py | 2 +- .../test_asyncio/test_eager_task_factory.py | 2 +- Lib/test/test_asyncio/test_events.py | 34 +++++++++---------- Lib/test/test_asyncio/test_free_threading.py | 2 +- Lib/test/test_asyncio/test_futures.py | 2 +- Lib/test/test_asyncio/test_futures2.py | 2 +- Lib/test/test_asyncio/test_graph.py | 2 +- Lib/test/test_asyncio/test_locks.py | 2 +- Lib/test/test_asyncio/test_pep492.py | 2 +- Lib/test/test_asyncio/test_proactor_events.py | 2 +- Lib/test/test_asyncio/test_protocols.py | 2 +- Lib/test/test_asyncio/test_queues.py | 2 +- Lib/test/test_asyncio/test_runners.py | 16 ++++----- Lib/test/test_asyncio/test_selector_events.py | 2 +- Lib/test/test_asyncio/test_sendfile.py | 2 +- Lib/test/test_asyncio/test_server.py | 2 +- Lib/test/test_asyncio/test_sock_lowlevel.py | 2 +- Lib/test/test_asyncio/test_ssl.py | 2 +- Lib/test/test_asyncio/test_sslproto.py | 2 +- Lib/test/test_asyncio/test_staggered.py | 2 +- Lib/test/test_asyncio/test_streams.py | 2 +- Lib/test/test_asyncio/test_subprocess.py | 2 +- Lib/test/test_asyncio/test_taskgroups.py | 2 +- Lib/test/test_asyncio/test_tasks.py | 2 +- Lib/test/test_asyncio/test_threads.py | 2 +- Lib/test/test_asyncio/test_timeouts.py | 2 +- Lib/test/test_asyncio/test_transports.py | 2 +- Lib/test/test_asyncio/test_unix_events.py | 2 +- Lib/test/test_asyncio/test_waitfor.py | 2 +- Lib/test/test_asyncio/test_windows_events.py | 14 ++++---- Lib/test/test_asyncio/test_windows_utils.py | 2 +- Lib/test/test_asyncio/utils.py | 6 ++++ .../test_interpreter_pool.py | 2 +- Lib/test/test_coroutines.py | 2 +- Lib/test/test_inspect/test_inspect.py | 2 +- Lib/test/test_logging.py | 6 ++-- Lib/test/test_os.py | 2 +- Lib/test/test_unittest/test_async_case.py | 6 ++-- Lib/test/test_unittest/testmock/testasync.py | 2 +- ...-05-25-11-02-05.gh-issue-134657.3YFhR9.rst | 1 + 47 files changed, 105 insertions(+), 90 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 4be7112fa017d4..32a5dbae03af21 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -51,15 +51,24 @@ def __getattr__(name: str): import warnings - deprecated = { - "AbstractEventLoopPolicy", - "DefaultEventLoopPolicy", - "WindowsSelectorEventLoopPolicy", - "WindowsProactorEventLoopPolicy", - } - if name in deprecated: - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - # deprecated things have underscores in front of them - return globals()["_" + name] + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2913f901dca65f..a7fb55982abe9c 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - "_AbstractEventLoopPolicy", "AbstractEventLoop", "AbstractServer", "Handle", "TimerHandle", - "_get_event_loop_policy", "get_event_loop_policy", - "_set_event_loop_policy", "set_event_loop_policy", "get_event_loop", "set_event_loop", @@ -791,7 +788,10 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import _DefaultEventLoopPolicy + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy _event_loop_policy = _DefaultEventLoopPolicy() diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f69c6a64c39ae6..1c1458127db5ac 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,6 @@ __all__ = ( 'SelectorEventLoop', - '_DefaultEventLoopPolicy', 'EventLoop', ) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index ffc29fa8dc686a..4cf1a075b30013 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -97,7 +97,7 @@ def get_asyncio_events__event_loop_policy(self): return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): asyncio = self.get_module('asyncio') - asyncio._set_event_loop_policy(policy) + asyncio.events._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index 2f2865bad2cdb1..fc3bd7dca62fd5 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -629,7 +629,7 @@ def setUp(self): def tearDown(self): self.loop.close() self.loop = None - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def check_async_iterator_anext(self, ait_class): with self.subTest(anext="pure-Python"): diff --git a/Lib/test/test_asyncio/test_base_events.py b/Lib/test/test_asyncio/test_base_events.py index 12179eb0c9e274..1b727f3b1febef 100644 --- a/Lib/test/test_asyncio/test_base_events.py +++ b/Lib/test/test_asyncio/test_base_events.py @@ -29,7 +29,7 @@ class CustomError(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def mock_socket_module(): diff --git a/Lib/test/test_asyncio/test_buffered_proto.py b/Lib/test/test_asyncio/test_buffered_proto.py index 9c386dd2e63815..6d3edcc36f5c79 100644 --- a/Lib/test/test_asyncio/test_buffered_proto.py +++ b/Lib/test/test_asyncio/test_buffered_proto.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ReceiveStuffProto(asyncio.BufferedProtocol): diff --git a/Lib/test/test_asyncio/test_context.py b/Lib/test/test_asyncio/test_context.py index ad394f44e7e5f6..f85f39839cbd79 100644 --- a/Lib/test/test_asyncio/test_context.py +++ b/Lib/test/test_asyncio/test_context.py @@ -4,7 +4,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipUnless(decimal.HAVE_CONTEXTVAR, "decimal is built with a thread-local context") diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 9f3b6f9acef3b6..da79ee9260a0e3 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -13,7 +13,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class EagerTaskFactoryLoopTests: diff --git a/Lib/test/test_asyncio/test_events.py b/Lib/test/test_asyncio/test_events.py index 873c503fa02b21..919d543b0329e9 100644 --- a/Lib/test/test_asyncio/test_events.py +++ b/Lib/test/test_asyncio/test_events.py @@ -38,7 +38,7 @@ from test.support import ALWAYS_EQ, LARGEST, SMALLEST def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def broken_unix_getsockname(): @@ -2843,13 +2843,13 @@ def test_default_event_loop_policy_deprecation(self): self.assertIsInstance(policy, asyncio.DefaultEventLoopPolicy) def test_event_loop_policy(self): - policy = asyncio._AbstractEventLoopPolicy() + policy = asyncio.events._AbstractEventLoopPolicy() self.assertRaises(NotImplementedError, policy.get_event_loop) self.assertRaises(NotImplementedError, policy.set_event_loop, object()) self.assertRaises(NotImplementedError, policy.new_event_loop) def test_get_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertIsNone(policy._local._loop) with self.assertRaises(RuntimeError): @@ -2857,7 +2857,7 @@ def test_get_event_loop(self): self.assertIsNone(policy._local._loop) def test_get_event_loop_does_not_call_set_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() with mock.patch.object( policy, "set_event_loop", @@ -2869,7 +2869,7 @@ def test_get_event_loop_does_not_call_set_event_loop(self): m_set_event_loop.assert_not_called() def test_get_event_loop_after_set_none(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() policy.set_event_loop(None) self.assertRaises(RuntimeError, policy.get_event_loop) @@ -2877,7 +2877,7 @@ def test_get_event_loop_after_set_none(self): def test_get_event_loop_thread(self, m_current_thread): def f(): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() self.assertRaises(RuntimeError, policy.get_event_loop) th = threading.Thread(target=f) @@ -2885,14 +2885,14 @@ def f(): th.join() def test_new_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() loop = policy.new_event_loop() self.assertIsInstance(loop, asyncio.AbstractEventLoop) loop.close() def test_set_event_loop(self): - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() old_loop = policy.new_event_loop() policy.set_event_loop(old_loop) @@ -2909,7 +2909,7 @@ def test_get_event_loop_policy(self): with self.assertWarnsRegex( DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): policy = asyncio.get_event_loop_policy() - self.assertIsInstance(policy, asyncio._AbstractEventLoopPolicy) + self.assertIsInstance(policy, asyncio.events._AbstractEventLoopPolicy) self.assertIs(policy, asyncio.get_event_loop_policy()) def test_set_event_loop_policy(self): @@ -2922,7 +2922,7 @@ def test_set_event_loop_policy(self): DeprecationWarning, "'asyncio.get_event_loop_policy' is deprecated"): old_policy = asyncio.get_event_loop_policy() - policy = asyncio._DefaultEventLoopPolicy() + policy = test_utils.DefaultEventLoopPolicy() with self.assertWarnsRegex( DeprecationWarning, "'asyncio.set_event_loop_policy' is deprecated"): asyncio.set_event_loop_policy(policy) @@ -3034,13 +3034,13 @@ def test_get_event_loop_returns_running_loop(self): class TestError(Exception): pass - class Policy(asyncio._DefaultEventLoopPolicy): + class Policy(test_utils.DefaultEventLoopPolicy): def get_event_loop(self): raise TestError - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio._set_event_loop_policy(Policy()) + asyncio.events._set_event_loop_policy(Policy()) loop = asyncio.new_event_loop() with self.assertRaises(TestError): @@ -3068,7 +3068,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() @@ -3078,9 +3078,9 @@ async def func(): self.assertIs(asyncio._get_running_loop(), None) def test_get_event_loop_returns_running_loop2(self): - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: - asyncio._set_event_loop_policy(asyncio._DefaultEventLoopPolicy()) + asyncio.events._set_event_loop_policy(test_utils.DefaultEventLoopPolicy()) loop = asyncio.new_event_loop() self.addCleanup(loop.close) @@ -3106,7 +3106,7 @@ async def func(): asyncio.get_event_loop() finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if loop is not None: loop.close() diff --git a/Lib/test/test_asyncio/test_free_threading.py b/Lib/test/test_asyncio/test_free_threading.py index 110996c348554c..d874ed00bd7e7a 100644 --- a/Lib/test/test_asyncio/test_free_threading.py +++ b/Lib/test/test_asyncio/test_free_threading.py @@ -15,7 +15,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestFreeThreading: diff --git a/Lib/test/test_asyncio/test_futures.py b/Lib/test/test_asyncio/test_futures.py index 8b51522278aaa6..785258a2749773 100644 --- a/Lib/test/test_asyncio/test_futures.py +++ b/Lib/test/test_asyncio/test_futures.py @@ -17,7 +17,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _fakefunc(f): diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py index e2cddea01ecd93..c7c0ebdac1b676 100644 --- a/Lib/test/test_asyncio/test_futures2.py +++ b/Lib/test/test_asyncio/test_futures2.py @@ -7,7 +7,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class FutureTests: diff --git a/Lib/test/test_asyncio/test_graph.py b/Lib/test/test_asyncio/test_graph.py index 62f6593c31d2d1..2f22fbccba42bc 100644 --- a/Lib/test/test_asyncio/test_graph.py +++ b/Lib/test/test_asyncio/test_graph.py @@ -5,7 +5,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def capture_test_stack(*, fut=None, depth=1): diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 047f03cbb14b56..e025d2990a3f8a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -20,7 +20,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class LockTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_pep492.py b/Lib/test/test_asyncio/test_pep492.py index 48f4a75e0fd56c..a0c8434c9457d2 100644 --- a/Lib/test/test_asyncio/test_pep492.py +++ b/Lib/test/test_asyncio/test_pep492.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # Test that asyncio.iscoroutine() uses collections.abc.Coroutine diff --git a/Lib/test/test_asyncio/test_proactor_events.py b/Lib/test/test_asyncio/test_proactor_events.py index 24c4e8546b17aa..b25daaface0807 100644 --- a/Lib/test/test_asyncio/test_proactor_events.py +++ b/Lib/test/test_asyncio/test_proactor_events.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def close_transport(transport): diff --git a/Lib/test/test_asyncio/test_protocols.py b/Lib/test/test_asyncio/test_protocols.py index 4484a031988533..29d3bd22705c6a 100644 --- a/Lib/test/test_asyncio/test_protocols.py +++ b/Lib/test/test_asyncio/test_protocols.py @@ -7,7 +7,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ProtocolsAbsTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_queues.py b/Lib/test/test_asyncio/test_queues.py index 090b9774c2289f..54bbe79f81ff69 100644 --- a/Lib/test/test_asyncio/test_queues.py +++ b/Lib/test/test_asyncio/test_queues.py @@ -6,7 +6,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class QueueBasicTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_runners.py b/Lib/test/test_asyncio/test_runners.py index 21f277bc2d8d5f..8a4d7f5c796661 100644 --- a/Lib/test/test_asyncio/test_runners.py +++ b/Lib/test/test_asyncio/test_runners.py @@ -12,14 +12,14 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def interrupt_self(): _thread.interrupt_main() -class TestPolicy(asyncio._AbstractEventLoopPolicy): +class TestPolicy(asyncio.events._AbstractEventLoopPolicy): def __init__(self, loop_factory): self.loop_factory = loop_factory @@ -61,15 +61,15 @@ def setUp(self): super().setUp() policy = TestPolicy(self.new_loop) - asyncio._set_event_loop_policy(policy) + asyncio.events._set_event_loop_policy(policy) def tearDown(self): - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() if policy.loop is not None: self.assertTrue(policy.loop.is_closed()) self.assertTrue(policy.loop.shutdown_ag_run) - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) super().tearDown() @@ -208,7 +208,7 @@ async def main(): await asyncio.sleep(0) return 42 - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() policy.set_event_loop = mock.Mock() asyncio.run(main()) self.assertTrue(policy.set_event_loop.called) @@ -259,7 +259,7 @@ def new_event_loop(): loop.set_task_factory(Task) return loop - asyncio._set_event_loop_policy(TestPolicy(new_event_loop)) + asyncio.events._set_event_loop_policy(TestPolicy(new_event_loop)) with self.assertRaises(asyncio.CancelledError): asyncio.run(main()) @@ -495,7 +495,7 @@ def test_set_event_loop_called_once(self): async def coro(): pass - policy = asyncio._get_event_loop_policy() + policy = asyncio.events._get_event_loop_policy() policy.set_event_loop = mock.Mock() runner = asyncio.Runner() runner.run(coro()) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index aab6a779170eb9..7b6d1bce5e460f 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestBaseSelectorEventLoop(BaseSelectorEventLoop): diff --git a/Lib/test/test_asyncio/test_sendfile.py b/Lib/test/test_asyncio/test_sendfile.py index e1b766d06cbe1e..dcd963b3355ef8 100644 --- a/Lib/test/test_asyncio/test_sendfile.py +++ b/Lib/test/test_asyncio/test_sendfile.py @@ -22,7 +22,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MySendfileProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_server.py b/Lib/test/test_asyncio/test_server.py index 32211f4cba32cb..5bd0f7e2af4f84 100644 --- a/Lib/test/test_asyncio/test_server.py +++ b/Lib/test/test_asyncio/test_server.py @@ -11,7 +11,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BaseStartServer(func_tests.FunctionalTestCaseMixin): diff --git a/Lib/test/test_asyncio/test_sock_lowlevel.py b/Lib/test/test_asyncio/test_sock_lowlevel.py index 4f7b9a1dda6b78..df4ec7948975f6 100644 --- a/Lib/test/test_asyncio/test_sock_lowlevel.py +++ b/Lib/test/test_asyncio/test_sock_lowlevel.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_ssl.py b/Lib/test/test_asyncio/test_ssl.py index 3a7185cd8974d0..06118f3a61587b 100644 --- a/Lib/test/test_asyncio/test_ssl.py +++ b/Lib/test/test_asyncio/test_ssl.py @@ -30,7 +30,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyBaseProto(asyncio.Protocol): diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index aa248c5786f634..3e304c166425b0 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index ad34aa6da01f54..32e4817b70d717 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 4fa4384346f9e9..f93ee54abc6469 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 341e3e979e002b..3a17c169c34f12 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -37,7 +37,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport): diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 0d69a436fdb840..91f6b03b4597a5 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -15,7 +15,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyExc(Exception): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index f6f976f213ac02..931a43816a257a 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) async def coroutine_function(): diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index c98c9a9b395ff9..8ad5f9b2c9e750 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 3ba84d63b2ca5f..f60722c48b7617 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index af10d3dc2a80df..dbb572e2e1536b 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index e020c1f3e4f677..22982dc9d8aefe 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -30,7 +30,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) MOCK_ANY = mock.ANY diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d083f6b4d2a535..dedc6bf69d770e 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # The following value can be used as a very small timeout: diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 69e9905205eee0..0af3368627afca 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -19,7 +19,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -330,16 +330,16 @@ def test_selector_win_policy(self): async def main(): self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -347,16 +347,16 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index a6b207567c4f00..97f078ff911b5a 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class PipeTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 0a96573a81c173..a480e16e81bb91 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -601,3 +601,9 @@ def func(): await asyncio.sleep(0) if exc is not None: raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index b10bbebd0984c4..d5c032d01cdf5d 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -512,7 +512,7 @@ def setUpClass(cls): # tests left a policy in place, just in case. policy = support.maybe_get_event_loop_policy() assert policy is None, policy - cls.addClassCleanup(lambda: asyncio._set_event_loop_policy(None)) + cls.addClassCleanup(lambda: asyncio.events._set_event_loop_policy(None)) def setUp(self): super().setUp() diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 954003ab14df93..a515e0f5ca9b5f 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -2337,7 +2337,7 @@ async def f(): pass finally: loop.close() - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) self.assertEqual(buffer, [1, 2, 'MyException']) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 5e1fcc1d3be989..55942c2823c492 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2820,7 +2820,7 @@ async def asyncTearDown(self): @classmethod def tearDownClass(cls): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index c46e6f8ed77f62..d8d1020a5a3f45 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5421,7 +5421,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5433,7 +5433,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5737,7 +5737,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) if handler: handler.close() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 5217037ae9d812..1e50dc43c35f5c 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -104,7 +104,7 @@ def create_file(filename, content=b'content'): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MiscTests(unittest.TestCase): diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013cfbf..91d45283eb3b1b 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -12,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -480,7 +480,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio._get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -490,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 0791675b5401ca..dc36ceeb6502e8 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class AsyncClass: diff --git a/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst b/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst new file mode 100644 index 00000000000000..1bf8ee504ef180 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst @@ -0,0 +1 @@ +:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. From 19518a4205d064ad6000525f2a3fae0d52bc05c9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 11:23:43 +0200 Subject: [PATCH 599/713] [3.14] gh-136447: Use `self.loop` instead of global `loop` variable in asyncio REPL (GH-136448) (#136458) gh-136447: Use `self.loop` instead of global `loop` variable in asyncio REPL (GH-136448) (cherry picked from commit 77fa7a4dcc771bf4d297ebfd4f357483d0750a1c) Co-authored-by: Justin Su --- Lib/asyncio/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 21ca5c5f62ae83..ff3a69d1e17297 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -64,7 +64,7 @@ def callback(): except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback, context=self.context) + self.loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() From 90717c2cecc20bfbcec7e7c39a99c42b7cbc5f90 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 15:59:40 +0200 Subject: [PATCH 600/713] [3.14] Docs: unittest.enterModuleContext is not a classmethod (GH-136464) (#136465) Co-authored-by: Geoffrey Thomas --- Doc/library/unittest.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index dcdda1719bf593..f8028075047cc3 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2556,7 +2556,7 @@ To add cleanup code that must be run even in the case of an exception, use .. versionadded:: 3.8 -.. classmethod:: enterModuleContext(cm) +.. function:: enterModuleContext(cm) Enter the supplied :term:`context manager`. If successful, also add its :meth:`~object.__exit__` method as a cleanup function by From 75640d4b1f65b316089f294c93745febabeffc87 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 16:12:47 +0200 Subject: [PATCH 601/713] [3.14] gh-131825: Fix `sqlite3` timezone-naive adapter recipe (GH-136270) (GH-136467) gh-131825: Fix `sqlite3` timezone-naive adapter recipe (GH-136270) (cherry picked from commit 6a6cd3c07c0300c8799878a48d555470be2a52f7) Co-authored-by: NekrodNIK <60639354+NekrodNIK@users.noreply.github.com> --- Doc/library/sqlite3.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e2726e53f09cd6..e939e61a9676ee 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -2289,7 +2289,7 @@ This section shows recipes for common adapters and converters. def adapt_datetime_iso(val): """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" - return val.isoformat() + return val.replace(tzinfo=None).isoformat() def adapt_datetime_epoch(val): """Adapt datetime.datetime to Unix timestamp.""" From 78359403c56cf2fa9461851da958ad044a34649d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 18:00:54 +0200 Subject: [PATCH 602/713] [3.14] gh-81520: Document unexpected `os.path.ismount` behaviour with btrfs subvolumes (GH-136058) (GH-136471) gh-81520: Document unexpected `os.path.ismount` behaviour with btrfs subvolumes (GH-136058) (cherry picked from commit 591abcc01fcf1c65c7fdfaca7274f5d3f9f022da) Co-authored-by: Oskar Roesler --- Doc/library/os.path.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index cd2c5dfbb3f122..32a2970d2d3a2c 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -298,9 +298,10 @@ the :mod:`glob` module.) device than *path*, or whether :file:`{path}/..` and *path* point to the same i-node on the same device --- this should detect mount points for all Unix and POSIX variants. It is not able to reliably detect bind mounts on the - same filesystem. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. + same filesystem. On Linux systems, it will always return ``True`` for btrfs + subvolumes, even if they aren't mount points. On Windows, a drive letter root + and a share UNC are always mount points, and for any other path + ``GetVolumePathName`` is called to see if it is different from the input path. .. versionchanged:: 3.4 Added support for detecting non-root mount points on Windows. From 4db567546adb4f7049dd8dcc1c6ea1601a000f0f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 9 Jul 2025 23:39:33 +0200 Subject: [PATCH 603/713] [3.14] gh-136145: Define 'standard library' and 'stdlib' in the glossary (GH-136485) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 92f392ad9e5e10ff98eac319e58ec79df5951ce0) Co-authored-by: Zachary Ware Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Éric --- Doc/glossary.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index c5c7994f1262a9..705b0a9279c6d4 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1280,6 +1280,16 @@ Glossary and ending with double underscores. Special methods are documented in :ref:`specialnames`. + standard library + The collection of :term:`packages `, :term:`modules ` + and :term:`extension modules ` distributed as a part + of the official Python interpreter package. The exact membership of the + collection may vary based on platform, available system libraries, or + other criteria. Documentation can be found at :ref:`library-index`. + + See also :data:`sys.stdlib_module_names` for a list of all possible + standard library module names. + statement A statement is part of a suite (a "block" of code). A statement is either an :term:`expression` or one of several constructs with a keyword, such @@ -1290,6 +1300,9 @@ Glossary issues such as incorrect types. See also :term:`type hints ` and the :mod:`typing` module. + stdlib + An abbreviation of :term:`standard library`. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong From b733bf7437555e24a36f66c98dcd8c771fd197a0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 01:13:29 +0200 Subject: [PATCH 604/713] [3.14] gh-102740: Clarify time.monotonic() "system-wide" in the doc (GH-136431) (#136488) gh-102740: Clarify time.monotonic() "system-wide" in the doc (GH-136431) (cherry picked from commit 9c4d28777526e9975b212d49fb0a530f773a3209) Co-authored-by: Victor Stinner --- Doc/library/time.rst | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 29b695a9b193d6..df9be68bf4f69d 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -306,10 +306,11 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.5 - The function is now always available and always system-wide. + The function is now always available and the clock is now the same for + all processes. .. versionchanged:: 3.10 - On macOS, the function is now system-wide. + On macOS, the clock is now the same for all processes. .. function:: monotonic_ns() -> int @@ -325,7 +326,8 @@ Functions Return the value (in fractional seconds) of a performance counter, i.e. a clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference + does include time elapsed during sleep. The clock is the same for all + processes. The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. @@ -340,7 +342,7 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.10 - On Windows, the function is now system-wide. + On Windows, the clock is now the same for all processes. .. versionchanged:: 3.13 Use the same clock as :func:`time.monotonic`. @@ -987,8 +989,8 @@ The following constant is the only parameter that can be sent to .. data:: CLOCK_REALTIME - System-wide real-time clock. Setting this clock requires appropriate - privileges. + Real-time clock. Setting this clock requires appropriate privileges. + The clock is the same for all processes. .. availability:: Unix. From 4d025a23188e0a8391cac16b264549b50af744ef Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 01:57:40 +0200 Subject: [PATCH 605/713] [3.14] gh-136476: Show the full stack in get_async_stack_trace in _remote_debugging (GH-136483) (#136490) gh-136476: Show the full stack in get_async_stack_trace in _remote_debugging (GH-136483) (cherry picked from commit ea45a2f97cb1d4774a6f88e63c6ce0a487f83031) Co-authored-by: Pablo Galindo Salgado --- Lib/test/test_external_inspection.py | 430 +++++-- ...-07-09-20-29-30.gh-issue-136476.HyLLzh.rst | 2 + Modules/_remote_debugging_module.c | 1032 ++++++++--------- Modules/clinic/_remote_debugging_module.c.h | 47 +- 4 files changed, 868 insertions(+), 643 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index 44890ebfe5f76d..a709b837161f48 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -8,7 +8,12 @@ import time from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry, requires_gil_enabled +from test.support import ( + os_helper, + SHORT_TIMEOUT, + busy_retry, + requires_gil_enabled, +) from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -236,55 +241,162 @@ def new_eager_loop(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["c2_root", "sub_main_1", "sub_main_2"]: + self.assertIn(task_name, tasks_names) + + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "c2_root": {"Task-1", "sub_main_1", "sub_main_2"}, + "Task-1": set(), + "sub_main_1": {"Task-1"}, + "sub_main_2": {"Task-1"}, + }, + ) - expected_stack_trace = [ - [ - FrameInfo([script_name, 10, "c5"]), - FrameInfo([script_name, 14, "c4"]), - FrameInfo([script_name, 17, "c3"]), - FrameInfo([script_name, 20, "c2"]), - ], - "c2_root", - [ - CoroInfo( - [ - [ - FrameInfo( + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ) + ], + "c2_root": [ + ( + tuple([script_name, 10, "c5"]), + tuple([script_name, 14, "c4"]), + tuple([script_name, 17, "c3"]), + tuple([script_name, 20, "c2"]), + ) + ], + "sub_main_1": [(tuple([script_name, 23, "c1"]),)], + "sub_main_2": [(tuple([script_name, 23, "c1"]),)], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "c2_root": [ + ( + "Task-1", + ( + tuple( [ taskgroups.__file__, ANY, "TaskGroup._aexit", ] ), - FrameInfo( + tuple( [ taskgroups.__file__, ANY, "TaskGroup.__aexit__", ] ), - FrameInfo([script_name, 26, "main"]), - ], + tuple([script_name, 26, "main"]), + ), + ), + ("sub_main_1", (tuple([script_name, 23, "c1"]),)), + ("sub_main_2", (tuple([script_name, 23, "c1"]),)), + ], + "sub_main_1": [ + ( "Task-1", - ] - ), - CoroInfo( - [ - [FrameInfo([script_name, 23, "c1"])], - "sub_main_1", - ] - ), - CoroInfo( - [ - [FrameInfo([script_name, 23, "c1"])], - "sub_main_2", - ] - ), - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + "sub_main_2": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -350,19 +462,29 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ + # For this simple asyncgen test, we only expect one task with the full coroutine stack + self.assertEqual(len(stack_trace[0].awaited_by), 1) + task = stack_trace[0].awaited_by[0] + self.assertEqual(task.task_name, "Task-1") + + # Check the coroutine stack - based on actual output, only shows main + coroutine_stack = sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + self.assertEqual( + coroutine_stack, [ - FrameInfo([script_name, 10, "gen_nested_call"]), - FrameInfo([script_name, 16, "gen"]), - FrameInfo([script_name, 19, "main"]), + ( + tuple([script_name, 10, "gen_nested_call"]), + tuple([script_name, 16, "gen"]), + tuple([script_name, 19, "main"]), + ) ], - "Task-1", - [], - ] - self.assertEqual(stack_trace, expected_stack_trace) + ) + + # No awaited_by relationships expected for this simple case + self.assertEqual(task.awaited_by, []) @skip_if_not_supported @unittest.skipIf( @@ -429,18 +551,73 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [ - FrameInfo([script_name, 11, "deep"]), - FrameInfo([script_name, 15, "c1"]), - ], - "Task-2", - [CoroInfo([[FrameInfo([script_name, 21, "main"])], "Task-1"])], + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [(tuple([script_name, 21, "main"]),)], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + ) + ], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ("Task-1", (tuple([script_name, 21, "main"]),)) + ], + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -510,36 +687,93 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - [ - FrameInfo([script_name, 11, "deep"]), - FrameInfo([script_name, 15, "c1"]), - FrameInfo( - [ - staggered.__file__, - ANY, - "staggered_race..run_one_coro", - ] - ), - ], - "Task-2", - [ - CoroInfo( - [ - [ - FrameInfo( + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple([staggered.__file__, ANY, "staggered_race"]), + tuple([script_name, 21, "main"]), + ) + ], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + tuple( + [ + staggered.__file__, + ANY, + "staggered_race..run_one_coro", + ] + ), + ) + ], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ( + "Task-1", + ( + tuple( [staggered.__file__, ANY, "staggered_race"] ), - FrameInfo([script_name, 21, "main"]), - ], - "Task-1", - ] - ) - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + tuple([script_name, 21, "main"]), + ), + ) + ], + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -973,7 +1207,10 @@ def main_work(): if not stack: continue current_frame = stack[0] - if current_frame.funcname == "main_work" and current_frame.lineno >15: + if ( + current_frame.funcname == "main_work" + and current_frame.lineno > 15 + ): found = True if found: @@ -981,7 +1218,9 @@ def main_work(): # Give a bit of time to take the next sample time.sleep(0.1) else: - self.fail("Main thread did not start its busy work on time") + self.fail( + "Main thread did not start its busy work on time" + ) # Get stack trace with only GIL holder unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) @@ -999,16 +1238,23 @@ def main_work(): p.wait(timeout=SHORT_TIMEOUT) # Verify we got multiple threads in all_traces - self.assertGreater(len(all_traces), 1, "Should have multiple threads") + self.assertGreater( + len(all_traces), 1, "Should have multiple threads" + ) # Verify we got exactly one thread in gil_traces - self.assertEqual(len(gil_traces), 1, "Should have exactly one GIL holder") + self.assertEqual( + len(gil_traces), 1, "Should have exactly one GIL holder" + ) # The GIL holder should be in the all_traces list gil_thread_id = gil_traces[0][0] all_thread_ids = [trace[0] for trace in all_traces] - self.assertIn(gil_thread_id, all_thread_ids, - "GIL holder should be among all threads") + self.assertIn( + gil_thread_id, + all_thread_ids, + "GIL holder should be among all threads", + ) if __name__ == "__main__": diff --git a/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst b/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst new file mode 100644 index 00000000000000..7634bd3be9346d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst @@ -0,0 +1,2 @@ +Fix a bug that was causing the ``get_async_stack_trace`` function to miss +some frames in the stack trace. diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 7b8d272e63b160..0dc797a9da82a0 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -257,20 +257,25 @@ is_frame_valid( uintptr_t code_object_addr ); -static int -parse_tasks_in_set( +typedef int (*thread_processor_func)( RemoteUnwinderObject *unwinder, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); + +typedef int (*set_entry_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context ); + static int parse_task( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *render_to, - int recurse_task + PyObject *render_to ); static int @@ -285,9 +290,27 @@ static int parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, + uintptr_t* address_of_code_object, uintptr_t* previous_frame ); +static int +parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +); + +static int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr); +static int read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *size); + +static int process_task_and_waiters(RemoteUnwinderObject *unwinder, uintptr_t task_addr, PyObject *result); +static int process_task_awaited_by(RemoteUnwinderObject *unwinder, uintptr_t task_address, set_entry_processor_func processor, void *context); +static int find_running_task_in_thread(RemoteUnwinderObject *unwinder, uintptr_t thread_state_addr, uintptr_t *running_task_addr); +static int get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr); +static int append_awaited_by(RemoteUnwinderObject *unwinder, unsigned long tid, uintptr_t head_addr, PyObject *result); + /* ============================================================================ * UTILITY FUNCTIONS AND HELPERS * ============================================================================ */ @@ -405,32 +428,149 @@ validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets) return 0; } -/* ============================================================================ - * MEMORY READING FUNCTIONS - * ============================================================================ */ +// Generic function to iterate through all threads +static int +iterate_threads( + RemoteUnwinderObject *unwinder, + thread_processor_func processor, + void *context +) { + uintptr_t thread_state_addr; + unsigned long tid = 0; -static inline int -read_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) -{ - int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(void*), ptr_addr); - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read pointer from remote memory"); + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + unwinder->interpreter_addr + unwinder->debug_offsets.interpreter_state.threads_main, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); return -1; } + + while (thread_state_addr != 0) { + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + unwinder->debug_offsets.thread_state.native_thread_id, + sizeof(tid), + &tid)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread ID"); + return -1; + } + + // Call the processor function for this thread + if (processor(unwinder, thread_state_addr, tid, context) < 0) { + return -1; + } + + // Move to next thread + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next thread state"); + return -1; + } + } + return 0; } -static inline int -read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *size) -{ - int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(Py_ssize_t), size); - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read Py_ssize_t from remote memory"); +// Generic function to iterate through set entries +static int +iterate_set_entries( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + set_entry_processor_func processor, + void *context +) { + char set_object[SIZEOF_SET_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, set_addr, + SIZEOF_SET_OBJ, set_object) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); return -1; } + + Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); + Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; + uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len && els < num_els) { + uintptr_t key_addr; + if (read_py_ptr(unwinder, table_ptr, &key_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry ref count"); + return -1; + } + + if (ref_cnt) { + // Process this valid set entry + if (processor(unwinder, key_addr, context) < 0) { + return -1; + } + els++; + } + } + table_ptr += sizeof(void*) * 2; + i++; + } + return 0; } +// Processor function for task waiters +static int +process_waiter_task( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *result = (PyObject *)context; + return process_task_and_waiters(unwinder, key_addr, result); +} + +// Processor function for parsing tasks in sets +static int +process_task_parser( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *awaited_by = (PyObject *)context; + return parse_task(unwinder, key_addr, awaited_by); +} + +/* ============================================================================ + * MEMORY READING FUNCTIONS + * ============================================================================ */ + +#define DEFINE_MEMORY_READER(type_name, c_type, error_msg) \ +static inline int \ +read_##type_name(RemoteUnwinderObject *unwinder, uintptr_t address, c_type *result) \ +{ \ + int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(c_type), result); \ + if (res < 0) { \ + set_exception_cause(unwinder, PyExc_RuntimeError, error_msg); \ + return -1; \ + } \ + return 0; \ +} + +DEFINE_MEMORY_READER(ptr, uintptr_t, "Failed to read pointer from remote memory") +DEFINE_MEMORY_READER(Py_ssize_t, Py_ssize_t, "Failed to read Py_ssize_t from remote memory") +DEFINE_MEMORY_READER(char, char, "Failed to read char from remote memory") + static int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) { @@ -442,17 +582,6 @@ read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_ad return 0; } -static int -read_char(RemoteUnwinderObject *unwinder, uintptr_t address, char *result) -{ - int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(char), result); - if (res < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read char from remote memory"); - return -1; - } - return 0; -} - /* ============================================================================ * PYTHON OBJECT READING FUNCTIONS * ============================================================================ */ @@ -799,39 +928,9 @@ parse_task_name( static int parse_task_awaited_by( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *awaited_by, - int recurse_task + PyObject *awaited_by ) { - // Read the entire TaskObj at once - char task_obj[SIZEOF_TASK_OBJ]; - if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, - unwinder->async_debug_offsets.asyncio_task_object.size, - task_obj) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object in awaited_by parsing"); - return -1; - } - - uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); - - if ((void*)task_ab_addr == NULL) { - return 0; - } - - char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); - - if (awaited_by_is_a_set) { - if (parse_tasks_in_set(unwinder, task_ab_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse tasks in awaited_by set"); - return -1; - } - } else { - if (parse_task(unwinder, task_ab_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse single awaited_by task"); - return -1; - } - } - - return 0; + return process_task_awaited_by(unwinder, task_address, process_task_parser, awaited_by); } static int @@ -940,12 +1039,13 @@ parse_coro_chain( // Parse the previous frame using the gi_iframe from local copy uintptr_t prev_frame; uintptr_t gi_iframe_addr = coro_address + unwinder->debug_offsets.gen_object.gi_iframe; - if (parse_frame_object(unwinder, &name, gi_iframe_addr, &prev_frame) < 0) { + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); return -1; } - if (name == NULL) { + if (!name) { return 0; } @@ -966,8 +1066,7 @@ parse_coro_chain( static PyObject* create_task_result( RemoteUnwinderObject *unwinder, - uintptr_t task_address, - int recurse_task + uintptr_t task_address ) { PyObject* result = NULL; PyObject *call_stack = NULL; @@ -983,11 +1082,7 @@ create_task_result( } // Create task name/address for second tuple element - if (recurse_task) { - tn = parse_task_name(unwinder, task_address); - } else { - tn = PyLong_FromUnsignedLongLong(task_address); - } + tn = PyLong_FromUnsignedLongLong(task_address); if (tn == NULL) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name/address"); goto error; @@ -1040,8 +1135,7 @@ static int parse_task( RemoteUnwinderObject *unwinder, uintptr_t task_address, - PyObject *render_to, - int recurse_task + PyObject *render_to ) { char is_task; PyObject* result = NULL; @@ -1057,7 +1151,7 @@ parse_task( } if (is_task) { - result = create_task_result(unwinder, task_address, recurse_task); + result = create_task_result(unwinder, task_address); if (!result) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); goto error; @@ -1084,11 +1178,11 @@ parse_task( PyStructSequence_SetItem(result, 0, empty_list); // This steals the reference PyStructSequence_SetItem(result, 1, task_name); // This steals the reference } - if (PyList_Append(render_to, result)) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task result to render list"); goto error; } + Py_DECREF(result); return 0; @@ -1098,234 +1192,335 @@ parse_task( } static int -process_set_entry( +process_single_task_node( RemoteUnwinderObject *unwinder, - uintptr_t table_ptr, - PyObject *awaited_by, - int recurse_task + uintptr_t task_addr, + PyObject **task_info, + PyObject *result ) { - uintptr_t key_addr; - if (read_py_ptr(unwinder, table_ptr, &key_addr)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); - return -1; + PyObject *tn = NULL; + PyObject *current_awaited_by = NULL; + PyObject *task_id = NULL; + PyObject *result_item = NULL; + PyObject *coroutine_stack = NULL; + + tn = parse_task_name(unwinder, task_addr); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); + goto error; } - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry reference count"); - return -1; - } + current_awaited_by = PyList_New(0); + if (current_awaited_by == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); + goto error; + } - if (ref_cnt) { - // if 'ref_cnt=0' it's a set dummy marker - if (parse_task(unwinder, key_addr, awaited_by, recurse_task)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task in set entry"); - return -1; - } - return 1; // Successfully processed a valid entry - } + // Extract the coroutine stack for this task + coroutine_stack = PyList_New(0); + if (coroutine_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); + goto error; } - return 0; // Entry was NULL or dummy marker -} -static int -parse_tasks_in_set( - RemoteUnwinderObject *unwinder, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task -) { - char set_object[SIZEOF_SET_OBJ]; - int err = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - set_addr, - SIZEOF_SET_OBJ, - set_object); - if (err < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); - return -1; + if (parse_task(unwinder, task_addr, coroutine_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); + goto error; } - Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); - Py_ssize_t set_len = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask) + 1; // The set contains the `mask+1` element slots. - uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + task_id = PyLong_FromUnsignedLongLong(task_addr); + if (task_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); + goto error; + } - Py_ssize_t i = 0; - Py_ssize_t els = 0; - while (i < set_len && els < num_els) { - int result = process_set_entry(unwinder, table_ptr, awaited_by, recurse_task); + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result_item = PyStructSequence_New(state->TaskInfo_Type); + if (result_item == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); + goto error; + } - if (result < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process set entry"); - return -1; - } - if (result > 0) { - els++; - } + PyStructSequence_SetItem(result_item, 0, task_id); // steals ref + PyStructSequence_SetItem(result_item, 1, tn); // steals ref + PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref + PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref - table_ptr += sizeof(void*) * 2; - i++; + // References transferred to tuple + task_id = NULL; + tn = NULL; + coroutine_stack = NULL; + current_awaited_by = NULL; + + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + return -1; + } + if (task_info != NULL) { + *task_info = result_item; + } + Py_DECREF(result_item); + + // Get back current_awaited_by reference for parse_task_awaited_by + current_awaited_by = PyStructSequence_GetItem(result_item, 3); + if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); + // No cleanup needed here since all references were transferred to result_item + // and result_item was already added to result list and decreffed + return -1; } + return 0; + +error: + Py_XDECREF(tn); + Py_XDECREF(current_awaited_by); + Py_XDECREF(task_id); + Py_XDECREF(result_item); + Py_XDECREF(coroutine_stack); + return -1; } +// Thread processor for get_all_awaited_by +static int +process_thread_for_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + uintptr_t head_addr = thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + return append_awaited_by(unwinder, tid, head_addr, result); +} +// Generic function to process task awaited_by static int -setup_async_result_structure(RemoteUnwinderObject *unwinder, PyObject **result, PyObject **calls) -{ - *result = PyList_New(1); - if (*result == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create async result structure"); +process_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + set_entry_processor_func processor, + void *context +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); return -1; } - *calls = PyList_New(0); - if (*calls == NULL) { - Py_DECREF(*result); - *result = NULL; - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create calls list in async result"); - return -1; + uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); + if ((void*)task_ab_addr == NULL) { + return 0; // No tasks waiting for this one } - if (PyList_SetItem(*result, 0, *calls)) { /* steals ref to 'calls' */ - Py_DECREF(*calls); - Py_DECREF(*result); - *result = NULL; - *calls = NULL; - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to set calls list in async result"); - return -1; - } + char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); - return 0; + if (awaited_by_is_a_set) { + return iterate_set_entries(unwinder, task_ab_addr, processor, context); + } else { + // Single task waiting + return processor(unwinder, task_ab_addr, context); + } } static int -add_task_info_to_result( +process_running_task_chain( RemoteUnwinderObject *unwinder, - PyObject *result, - uintptr_t running_task_addr + uintptr_t running_task_addr, + uintptr_t thread_state_addr, + PyObject *result ) { - PyObject *tn = parse_task_name(unwinder, running_task_addr); - if (tn == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name for result"); + uintptr_t running_task_code_obj = 0; + if(get_task_code_object(unwinder, running_task_addr, &running_task_code_obj) < 0) { + return -1; + } + + // First, add this task to the result + PyObject *task_info = NULL; + if (process_single_task_node(unwinder, running_task_addr, &task_info, result) < 0) { return -1; } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task name to result"); + // Get the chain from the current frame to this task + PyObject *coro_chain = PyStructSequence_GET_ITEM(task_info, 2); + assert(coro_chain != NULL); + if (PyList_GET_SIZE(coro_chain) != 1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Coro chain is not a single item"); return -1; } - Py_DECREF(tn); + PyObject *coro_info = PyList_GET_ITEM(coro_chain, 0); + assert(coro_info != NULL); + PyObject *frame_chain = PyStructSequence_GET_ITEM(coro_info, 0); + assert(frame_chain != NULL); - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list for result"); + // Clear the coro_chain + if (PyList_Clear(frame_chain) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to clear coroutine chain"); return -1; } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by to result"); + // Add the chain from the current frame to this task + if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_code_obj) < 0) { return -1; } - Py_DECREF(awaited_by); - if (parse_task_awaited_by( - unwinder, running_task_addr, awaited_by, 1) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by for result"); + // Now find all tasks that are waiting for this task and process them + if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, result) < 0) { return -1; } return 0; } +// Thread processor for get_async_stack_trace static int -process_single_task_node( +process_thread_for_async_stack_trace( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + + // Find running task in this thread + uintptr_t running_task_addr; + if (find_running_task_in_thread(unwinder, thread_state_addr, &running_task_addr) < 0) { + return 0; + } + + // If we found a running task, process it and its waiters + if ((void*)running_task_addr != NULL) { + PyObject *task_list = PyList_New(0); + if (task_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create task list for thread"); + return -1; + } + + if (process_running_task_chain(unwinder, running_task_addr, thread_state_addr, task_list) < 0) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process running task chain"); + return -1; + } + + // Create AwaitedInfo structure for this thread + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + return -1; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *awaited_info = PyStructSequence_New(state->AwaitedInfo_Type); + if (awaited_info == NULL) { + Py_DECREF(tid_py); + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } + + PyStructSequence_SetItem(awaited_info, 0, tid_py); // steals ref + PyStructSequence_SetItem(awaited_info, 1, task_list); // steals ref + + if (PyList_Append(result, awaited_info)) { + Py_DECREF(awaited_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append AwaitedInfo to result"); + return -1; + } + Py_DECREF(awaited_info); + } + + return 0; +} + +static int +process_task_and_waiters( RemoteUnwinderObject *unwinder, uintptr_t task_addr, PyObject *result ) { - PyObject *tn = NULL; - PyObject *current_awaited_by = NULL; - PyObject *task_id = NULL; - PyObject *result_item = NULL; - PyObject *coroutine_stack = NULL; - - tn = parse_task_name(unwinder, task_addr); - if (tn == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); - goto error; + // First, add this task to the result + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { + return -1; } - current_awaited_by = PyList_New(0); - if (current_awaited_by == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); - goto error; + // Now find all tasks that are waiting for this task and process them + return process_task_awaited_by(unwinder, task_addr, process_waiter_task, result); +} + +static int +find_running_task_in_thread( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; + + uintptr_t address_of_running_loop; + int bytes_read = read_py_ptr( + unwinder, + thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); + return -1; } - // Extract the coroutine stack for this task - coroutine_stack = PyList_New(0); - if (coroutine_stack == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); - goto error; + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; } - if (parse_task(unwinder, task_addr, coroutine_stack, 0) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); - goto error; + int err = read_ptr( + unwinder, + thread_state_addr + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); + return -1; } - task_id = PyLong_FromUnsignedLongLong(task_addr); - if (task_id == NULL) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); - goto error; + return 0; +} + +static int +get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr) { + uintptr_t running_coro_addr = 0; + + if(read_py_ptr( + unwinder, + task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, + &running_coro_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); + return -1; } - RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); - result_item = PyStructSequence_New(state->TaskInfo_Type); - if (result_item == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); - goto error; + if (running_coro_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); + return -1; } - PyStructSequence_SetItem(result_item, 0, task_id); // steals ref - PyStructSequence_SetItem(result_item, 1, tn); // steals ref - PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref - PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref - - // References transferred to tuple - task_id = NULL; - tn = NULL; - coroutine_stack = NULL; - current_awaited_by = NULL; - - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + if (read_py_ptr( + unwinder, + running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); return -1; } - Py_DECREF(result_item); - // Get back current_awaited_by reference for parse_task_awaited_by - current_awaited_by = PyStructSequence_GetItem(result_item, 3); - if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by, 0) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); - // No cleanup needed here since all references were transferred to result_item - // and result_item was already added to result list and decreffed + if (*code_obj_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); return -1; } return 0; - -error: - Py_XDECREF(tn); - Py_XDECREF(current_awaited_by); - Py_XDECREF(task_id); - Py_XDECREF(result_item); - Py_XDECREF(coroutine_stack); - return -1; } /* ============================================================================ @@ -1908,45 +2103,13 @@ populate_initial_state_data( return 0; } + static int find_running_frame( RemoteUnwinderObject *unwinder, - uintptr_t runtime_start_address, + uintptr_t address_of_thread, uintptr_t *frame ) { - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state for running frame"); - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL in running frame search"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_main, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread address for running frame"); - return -1; - } - - // No Python frames are available for us (can happen at tear-down). if ((void*)address_of_thread != NULL) { int err = read_ptr( unwinder, @@ -1963,133 +2126,6 @@ find_running_frame( return 0; } -static int -find_running_task( - RemoteUnwinderObject *unwinder, - uintptr_t *running_task_addr -) { - *running_task_addr = (uintptr_t)NULL; - - uint64_t interpreter_state_list_head = - unwinder->debug_offsets.runtime_state.interpreters_head; - - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - unwinder->runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state for running task"); - return -1; - } - - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL in running task search"); - return -1; - } - - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address_of_interpreter_state + - unwinder->debug_offsets.interpreter_state.threads_head, - sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread head for running task"); - return -1; - } - - uintptr_t address_of_running_loop; - // No Python frames are available for us (can happen at tear-down). - if ((void*)address_of_thread == NULL) { - return 0; - } - - bytes_read = read_py_ptr( - unwinder, - address_of_thread - + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, - &address_of_running_loop); - if (bytes_read == -1) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); - return -1; - } - - // no asyncio loop is now running - if ((void*)address_of_running_loop == NULL) { - return 0; - } - - int err = read_ptr( - unwinder, - address_of_thread - + unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, - running_task_addr); - if (err) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); - return -1; - } - - return 0; -} - -static int -find_running_task_and_coro( - RemoteUnwinderObject *unwinder, - uintptr_t *running_task_addr, - uintptr_t *running_coro_addr, - uintptr_t *running_task_code_obj -) { - *running_task_addr = (uintptr_t)NULL; - if (find_running_task( - unwinder, running_task_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task search failed"); - return -1; - } - - if ((void*)*running_task_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No running task found"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task address is NULL"); - return -1; - } - - if (read_py_ptr( - unwinder, - *running_task_addr + unwinder->async_debug_offsets.asyncio_task_object.task_coro, - running_coro_addr) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); - return -1; - } - - if ((void*)*running_coro_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); - return -1; - } - - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - if (read_py_ptr( - unwinder, - *running_coro_addr + unwinder->debug_offsets.gen_object.gi_iframe, - running_task_code_obj) < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); - return -1; - } - - if ((void*)*running_task_code_obj == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); - return -1; - } - - return 0; -} - - /* ============================================================================ * FRAME PARSING FUNCTIONS * ============================================================================ */ @@ -2126,9 +2162,11 @@ parse_frame_object( RemoteUnwinderObject *unwinder, PyObject** result, uintptr_t address, + uintptr_t* address_of_code_object, uintptr_t* previous_frame ) { char frame[SIZEOF_INTERP_FRAME]; + *address_of_code_object = 0; Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( &unwinder->handle, @@ -2158,77 +2196,38 @@ parse_frame_object( } #endif - return parse_code_object(unwinder, result, code_object,instruction_pointer, previous_frame, tlbc_index); -} - -static int -parse_async_frame_object( - RemoteUnwinderObject *unwinder, - PyObject** result, - uintptr_t address, - uintptr_t* previous_frame, - uintptr_t* code_object -) { - char frame[SIZEOF_INTERP_FRAME]; - - Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( - &unwinder->handle, - address, - SIZEOF_INTERP_FRAME, - frame - ); - if (bytes_read < 0) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read async frame"); - return -1; - } - - *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); - *code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); - int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, *code_object); - if (frame_valid != 1) { - return frame_valid; - } - - uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); - - // Get tlbc_index for free threading builds - int32_t tlbc_index = 0; -#ifdef Py_GIL_DISABLED - if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { - tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); - } -#endif - - if (parse_code_object( - unwinder, result, *code_object, instruction_pointer, previous_frame, tlbc_index)) { - set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse code object in async frame"); - return -1; - } - - return 1; + *address_of_code_object = code_object; + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); } static int -parse_async_frame_chain( + parse_async_frame_chain( RemoteUnwinderObject *unwinder, PyObject *calls, + uintptr_t address_of_thread, uintptr_t running_task_code_obj ) { uintptr_t address_of_current_frame; - if (find_running_frame(unwinder, unwinder->runtime_start_address, &address_of_current_frame) < 0) { + if (find_running_frame(unwinder, address_of_thread, &address_of_current_frame) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Running frame search failed in async chain"); return -1; } - uintptr_t address_of_code_object; + PyObject *frames = PyList_New(0); + if (frames == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create frames list"); + return -1; + } + while ((void*)address_of_current_frame != NULL) { PyObject* frame_info = NULL; - int res = parse_async_frame_object( + uintptr_t address_of_code_object; + int res = parse_frame_object( unwinder, &frame_info, address_of_current_frame, - &address_of_current_frame, - &address_of_code_object + &address_of_code_object, + &address_of_current_frame ); if (res < 0) { @@ -2294,7 +2293,7 @@ append_awaited_by_for_thread( uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) - unwinder->async_debug_offsets.asyncio_task_object.task_node; - if (process_single_task_node(unwinder, task_addr, result) < 0) { + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); return -1; } @@ -2389,7 +2388,8 @@ process_frame_chain( // Try chunks first, fallback to direct memory read if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { PyErr_Clear(); - if (parse_frame_object(unwinder, &frame, frame_addr, &next_frame_addr) < 0) { + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); return -1; } @@ -2830,53 +2830,12 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s goto result_err; } - uintptr_t thread_state_addr; - unsigned long tid = 0; - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - self->interpreter_addr - + self->debug_offsets.interpreter_state.threads_main, - sizeof(void*), - &thread_state_addr)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read main thread state in get_all_awaited_by"); + // Process all threads + if (iterate_threads(self, process_thread_for_awaited_by, result) < 0) { goto result_err; } - uintptr_t head_addr; - while (thread_state_addr != 0) { - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - thread_state_addr - + self->debug_offsets.thread_state.native_thread_id, - sizeof(tid), - &tid)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read thread ID in get_all_awaited_by"); - goto result_err; - } - - head_addr = thread_state_addr - + self->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; - - if (append_awaited_by(self, tid, head_addr, result)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to append awaited_by for thread in get_all_awaited_by"); - goto result_err; - } - - if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( - &self->handle, - thread_state_addr + self->debug_offsets.thread_state.next, - sizeof(void*), - &thread_state_addr)) - { - set_exception_cause(self, PyExc_RuntimeError, "Failed to read next thread state in get_all_awaited_by"); - goto result_err; - } - } - - head_addr = self->interpreter_addr + uintptr_t head_addr = self->interpreter_addr + self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; // On top of a per-thread task lists used by default by asyncio to avoid @@ -2903,32 +2862,50 @@ _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *s @critical_section _remote_debugging.RemoteUnwinder.get_async_stack_trace -Returns information about the currently running async task and its stack trace. +Get the currently running async tasks and their dependency graphs from the remote process. -Returns a tuple of (task_info, stack_frames) where: -- task_info is a tuple of (task_id, task_name) identifying the task -- stack_frames is a list of tuples (function_name, filename, line_number) representing - the Python stack frames for the task, ordered from most recent to oldest +This returns information about running tasks and all tasks that are waiting for them, +forming a complete dependency graph for each thread's active task. -Example: - ((4345585712, 'Task-1'), [ - ('run_echo_server', 'server.py', 127), - ('serve_forever', 'server.py', 45), - ('main', 'app.py', 23) - ]) +For each thread with a running task, returns the running task plus all tasks that +transitively depend on it (tasks waiting for the running task, tasks waiting for +those tasks, etc.). + +Returns a list of per-thread results, where each thread result contains: +- Thread ID +- List of task information for the running task and all its waiters + +Each task info contains: +- Task ID (memory address) +- Task name +- Call stack frames: List of (func_name, filename, lineno) +- List of tasks waiting for this task (recursive structure) Raises: RuntimeError: If AsyncioDebug section is not available in the target process - RuntimeError: If there is an error copying memory from the target process - OSError: If there is an error accessing the target process - PermissionError: If access to the target process is denied - UnicodeDecodeError: If there is an error decoding strings from the target process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output (similar structure to get_all_awaited_by but only for running tasks): +[ + # Thread 140234 results + (140234, [ + # Running task and its complete waiter dependency graph + (4345585712, 'main_task', + [("run_server", "server.py", 127), ("main", "app.py", 23)], + [ + # Tasks waiting for main_task + (4345585800, 'worker_1', [...], [...]), + (4345585900, 'worker_2', [...], [...]) + ]) + ]) +] [clinic start generated code]*/ static PyObject * _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) -/*[clinic end generated code: output=6433d52b55e87bbe input=11b7150c59d4c60f]*/ +/*[clinic end generated code: output=6433d52b55e87bbe input=8744b47c9ec2220a]*/ { if (!self->async_debug_offsets_available) { PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); @@ -2936,35 +2913,20 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject return NULL; } - PyObject *result = NULL; - PyObject *calls = NULL; - - if (setup_async_result_structure(self, &result, &calls) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to setup async result structure"); - goto cleanup; - } - - uintptr_t running_task_addr, running_coro_addr, running_task_code_obj; - if (find_running_task_and_coro(self, &running_task_addr, - &running_coro_addr, &running_task_code_obj) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to find running task and coro"); - goto cleanup; - } - - if (parse_async_frame_chain(self, calls, running_task_code_obj) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to parse async frame chain"); - goto cleanup; + PyObject *result = PyList_New(0); + if (result == NULL) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create result list in get_async_stack_trace"); + return NULL; } - if (add_task_info_to_result(self, result, running_task_addr) < 0) { - set_exception_cause(self, PyExc_RuntimeError, "Failed to add task info to result"); - goto cleanup; + // Process all threads + if (iterate_threads(self, process_thread_for_async_stack_trace, result) < 0) { + goto result_err; } _Py_RemoteDebug_ClearCache(&self->handle); return result; - -cleanup: +result_err: _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); return NULL; diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h index e80b24b54c0ffa..f6a51cdba6b401 100644 --- a/Modules/clinic/_remote_debugging_module.c.h +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -235,26 +235,41 @@ PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, "get_async_stack_trace($self, /)\n" "--\n" "\n" -"Returns information about the currently running async task and its stack trace.\n" +"Get the currently running async tasks and their dependency graphs from the remote process.\n" "\n" -"Returns a tuple of (task_info, stack_frames) where:\n" -"- task_info is a tuple of (task_id, task_name) identifying the task\n" -"- stack_frames is a list of tuples (function_name, filename, line_number) representing\n" -" the Python stack frames for the task, ordered from most recent to oldest\n" +"This returns information about running tasks and all tasks that are waiting for them,\n" +"forming a complete dependency graph for each thread\'s active task.\n" "\n" -"Example:\n" -" ((4345585712, \'Task-1\'), [\n" -" (\'run_echo_server\', \'server.py\', 127),\n" -" (\'serve_forever\', \'server.py\', 45),\n" -" (\'main\', \'app.py\', 23)\n" -" ])\n" +"For each thread with a running task, returns the running task plus all tasks that\n" +"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" +"those tasks, etc.).\n" +"\n" +"Returns a list of per-thread results, where each thread result contains:\n" +"- Thread ID\n" +"- List of task information for the running task and all its waiters\n" +"\n" +"Each task info contains:\n" +"- Task ID (memory address)\n" +"- Task name\n" +"- Call stack frames: List of (func_name, filename, lineno)\n" +"- List of tasks waiting for this task (recursive structure)\n" "\n" "Raises:\n" " RuntimeError: If AsyncioDebug section is not available in the target process\n" -" RuntimeError: If there is an error copying memory from the target process\n" -" OSError: If there is an error accessing the target process\n" -" PermissionError: If access to the target process is denied\n" -" UnicodeDecodeError: If there is an error decoding strings from the target process"); +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"[\n" +" (140234, [\n" +" (4345585712, \'main_task\',\n" +" [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n" +" [\n" +" (4345585800, \'worker_1\', [...], [...]),\n" +" (4345585900, \'worker_2\', [...], [...])\n" +" ])\n" +" ])\n" +"]"); #define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, @@ -273,4 +288,4 @@ _remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject return return_value; } -/*[clinic end generated code: output=a37ab223d5081b16 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0dd1e6e8bab2a8b1 input=a9049054013a1b77]*/ From 7937f16adeac8efb9ce3e63e35537da6c94f67c1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 02:51:09 +0200 Subject: [PATCH 606/713] [3.14] gh-135846: Add zstd dependency to Android build script (GH-136253) (#136491) Adds zstd to the Android build process. (cherry picked from commit 61dd9fdad729fe02d91c03804659f7d0c5a89276) Co-authored-by: Emma Smith Co-authored-by: Malcolm Smith --- Android/android.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Android/android.py b/Android/android.py index 551168fc4b2f5a..a3a48c0c6b7027 100755 --- a/Android/android.py +++ b/Android/android.py @@ -175,7 +175,7 @@ def unpack_deps(host, prefix_dir): os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" for name_ver in ["bzip2-1.0.8-3", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1"]: + "sqlite-3.49.1-0", "xz-5.4.6-1", "zstd-1.5.7-1"]: filename = f"{name_ver}-{host}.tar.gz" download(f"{deps_url}/{name_ver}/{filename}") shutil.unpack_archive(filename) From 78804219832a3965784093c76132beeadc762dc0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 10:36:48 +0200 Subject: [PATCH 607/713] [3.14] gh-136476: Remove creation of unused list (GH-136494) (GH-136495) (cherry picked from commit b44316a0976fb3fcd50bae9d67b0810ee0252d93) Co-authored-by: Petr Viktorin --- Modules/_remote_debugging_module.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 0dc797a9da82a0..a2619de0510b59 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -2213,12 +2213,6 @@ static int return -1; } - PyObject *frames = PyList_New(0); - if (frames == NULL) { - set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create frames list"); - return -1; - } - while ((void*)address_of_current_frame != NULL) { PyObject* frame_info = NULL; uintptr_t address_of_code_object; From 2de82bcc62ddcaae7af0f53f59cb019a791a61d3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 11:30:07 +0200 Subject: [PATCH 608/713] [3.14] gh-136438: Make sure `test_builtins` pass with all optimization levels (GH-136474) (#136496) gh-136438: Make sure `test_builtins` pass with all optimization levels (GH-136474) (cherry picked from commit c17654334946b232aa296696cf70ec93a09d8156) Co-authored-by: sobolevn --- Lib/test/test_builtin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 14fe3355239615..8830641f0abdc7 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -436,7 +436,7 @@ def f(): """doc""" # test both direct compilation and compilation via AST codeobjs = [] codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) + tree = ast.parse(codestr, optimize=optval) codeobjs.append(compile(tree, "", "exec", optimize=optval)) for code in codeobjs: ns = {} @@ -624,7 +624,7 @@ def test_compile_ast(self): for opt in [opt1, opt2]: opt_right = opt.value.right self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, True) + self.assertEqual(opt_right.value, __debug__) def test_delattr(self): sys.spam = 1 From 1346049782daca649a0e6f6993e085c7a6637772 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:10:43 +0200 Subject: [PATCH 609/713] [3.14] gh-136209: Add .. c:var:: declarations for C exception types (GH-136210) (GH-136504) (cherry picked from commit 85bc89f35f40c844df74d913fd32b2b1475fc942) Co-authored-by: Petr Viktorin Co-authored-by: Victor Stinner --- Doc/c-api/exceptions.rst | 438 +++++++++++++++++---------------------- Doc/conf.py | 69 ------ 2 files changed, 187 insertions(+), 320 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a750cda3e2d474..3ff4631a8e53c4 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -982,184 +982,135 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. .. _standardexceptions: -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException (C var) - single: PyExc_BaseExceptionGroup (C var) - single: PyExc_Exception (C var) - single: PyExc_ArithmeticError (C var) - single: PyExc_AssertionError (C var) - single: PyExc_AttributeError (C var) - single: PyExc_BlockingIOError (C var) - single: PyExc_BrokenPipeError (C var) - single: PyExc_BufferError (C var) - single: PyExc_ChildProcessError (C var) - single: PyExc_ConnectionAbortedError (C var) - single: PyExc_ConnectionError (C var) - single: PyExc_ConnectionRefusedError (C var) - single: PyExc_ConnectionResetError (C var) - single: PyExc_EOFError (C var) - single: PyExc_FileExistsError (C var) - single: PyExc_FileNotFoundError (C var) - single: PyExc_FloatingPointError (C var) - single: PyExc_GeneratorExit (C var) - single: PyExc_ImportError (C var) - single: PyExc_IndentationError (C var) - single: PyExc_IndexError (C var) - single: PyExc_InterruptedError (C var) - single: PyExc_IsADirectoryError (C var) - single: PyExc_KeyError (C var) - single: PyExc_KeyboardInterrupt (C var) - single: PyExc_LookupError (C var) - single: PyExc_MemoryError (C var) - single: PyExc_ModuleNotFoundError (C var) - single: PyExc_NameError (C var) - single: PyExc_NotADirectoryError (C var) - single: PyExc_NotImplementedError (C var) - single: PyExc_OSError (C var) - single: PyExc_OverflowError (C var) - single: PyExc_PermissionError (C var) - single: PyExc_ProcessLookupError (C var) - single: PyExc_PythonFinalizationError (C var) - single: PyExc_RecursionError (C var) - single: PyExc_ReferenceError (C var) - single: PyExc_RuntimeError (C var) - single: PyExc_StopAsyncIteration (C var) - single: PyExc_StopIteration (C var) - single: PyExc_SyntaxError (C var) - single: PyExc_SystemError (C var) - single: PyExc_SystemExit (C var) - single: PyExc_TabError (C var) - single: PyExc_TimeoutError (C var) - single: PyExc_TypeError (C var) - single: PyExc_UnboundLocalError (C var) - single: PyExc_UnicodeDecodeError (C var) - single: PyExc_UnicodeEncodeError (C var) - single: PyExc_UnicodeError (C var) - single: PyExc_UnicodeTranslateError (C var) - single: PyExc_ValueError (C var) - single: PyExc_ZeroDivisionError (C var) - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BaseExceptionGroup` | :exc:`BaseExceptionGroup` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ +Exception and warning types +=========================== + +All standard Python exceptions and warning categories are available as global +variables whose names are ``PyExc_`` followed by the Python exception name. +These have the type :c:expr:`PyObject*`; they are all class objects. + +For completeness, here are all the variables: + +Exception types +--------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_BaseException + * :exc:`BaseException` + * * .. c:var:: PyObject *PyExc_BaseExceptionGroup + * :exc:`BaseExceptionGroup` + * * .. c:var:: PyObject *PyExc_Exception + * :exc:`Exception` + * * .. c:var:: PyObject *PyExc_ArithmeticError + * :exc:`ArithmeticError` + * * .. c:var:: PyObject *PyExc_AssertionError + * :exc:`AssertionError` + * * .. c:var:: PyObject *PyExc_AttributeError + * :exc:`AttributeError` + * * .. c:var:: PyObject *PyExc_BlockingIOError + * :exc:`BlockingIOError` + * * .. c:var:: PyObject *PyExc_BrokenPipeError + * :exc:`BrokenPipeError` + * * .. c:var:: PyObject *PyExc_BufferError + * :exc:`BufferError` + * * .. c:var:: PyObject *PyExc_ChildProcessError + * :exc:`ChildProcessError` + * * .. c:var:: PyObject *PyExc_ConnectionAbortedError + * :exc:`ConnectionAbortedError` + * * .. c:var:: PyObject *PyExc_ConnectionError + * :exc:`ConnectionError` + * * .. c:var:: PyObject *PyExc_ConnectionRefusedError + * :exc:`ConnectionRefusedError` + * * .. c:var:: PyObject *PyExc_ConnectionResetError + * :exc:`ConnectionResetError` + * * .. c:var:: PyObject *PyExc_EOFError + * :exc:`EOFError` + * * .. c:var:: PyObject *PyExc_FileExistsError + * :exc:`FileExistsError` + * * .. c:var:: PyObject *PyExc_FileNotFoundError + * :exc:`FileNotFoundError` + * * .. c:var:: PyObject *PyExc_FloatingPointError + * :exc:`FloatingPointError` + * * .. c:var:: PyObject *PyExc_GeneratorExit + * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportError + * :exc:`ImportError` + * * .. c:var:: PyObject *PyExc_IndentationError + * :exc:`IndentationError` + * * .. c:var:: PyObject *PyExc_IndexError + * :exc:`IndexError` + * * .. c:var:: PyObject *PyExc_InterruptedError + * :exc:`InterruptedError` + * * .. c:var:: PyObject *PyExc_IsADirectoryError + * :exc:`IsADirectoryError` + * * .. c:var:: PyObject *PyExc_KeyError + * :exc:`KeyError` + * * .. c:var:: PyObject *PyExc_KeyboardInterrupt + * :exc:`KeyboardInterrupt` + * * .. c:var:: PyObject *PyExc_LookupError + * :exc:`LookupError` + * * .. c:var:: PyObject *PyExc_MemoryError + * :exc:`MemoryError` + * * .. c:var:: PyObject *PyExc_ModuleNotFoundError + * :exc:`ModuleNotFoundError` + * * .. c:var:: PyObject *PyExc_NameError + * :exc:`NameError` + * * .. c:var:: PyObject *PyExc_NotADirectoryError + * :exc:`NotADirectoryError` + * * .. c:var:: PyObject *PyExc_NotImplementedError + * :exc:`NotImplementedError` + * * .. c:var:: PyObject *PyExc_OSError + * :exc:`OSError` + * * .. c:var:: PyObject *PyExc_OverflowError + * :exc:`OverflowError` + * * .. c:var:: PyObject *PyExc_PermissionError + * :exc:`PermissionError` + * * .. c:var:: PyObject *PyExc_ProcessLookupError + * :exc:`ProcessLookupError` + * * .. c:var:: PyObject *PyExc_PythonFinalizationError + * :exc:`PythonFinalizationError` + * * .. c:var:: PyObject *PyExc_RecursionError + * :exc:`RecursionError` + * * .. c:var:: PyObject *PyExc_ReferenceError + * :exc:`ReferenceError` + * * .. c:var:: PyObject *PyExc_RuntimeError + * :exc:`RuntimeError` + * * .. c:var:: PyObject *PyExc_StopAsyncIteration + * :exc:`StopAsyncIteration` + * * .. c:var:: PyObject *PyExc_StopIteration + * :exc:`StopIteration` + * * .. c:var:: PyObject *PyExc_SyntaxError + * :exc:`SyntaxError` + * * .. c:var:: PyObject *PyExc_SystemError + * :exc:`SystemError` + * * .. c:var:: PyObject *PyExc_SystemExit + * :exc:`SystemExit` + * * .. c:var:: PyObject *PyExc_TabError + * :exc:`TabError` + * * .. c:var:: PyObject *PyExc_TimeoutError + * :exc:`TimeoutError` + * * .. c:var:: PyObject *PyExc_TypeError + * :exc:`TypeError` + * * .. c:var:: PyObject *PyExc_UnboundLocalError + * :exc:`UnboundLocalError` + * * .. c:var:: PyObject *PyExc_UnicodeDecodeError + * :exc:`UnicodeDecodeError` + * * .. c:var:: PyObject *PyExc_UnicodeEncodeError + * :exc:`UnicodeEncodeError` + * * .. c:var:: PyObject *PyExc_UnicodeError + * :exc:`UnicodeError` + * * .. c:var:: PyObject *PyExc_UnicodeTranslateError + * :exc:`UnicodeTranslateError` + * * .. c:var:: PyObject *PyExc_ValueError + * :exc:`ValueError` + * * .. c:var:: PyObject *PyExc_ZeroDivisionError + * :exc:`ZeroDivisionError` .. versionadded:: 3.3 :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, @@ -1180,94 +1131,79 @@ the variables: .. versionadded:: 3.11 :c:data:`PyExc_BaseExceptionGroup`. -These are compatibility aliases to :c:data:`PyExc_OSError`: -.. index:: - single: PyExc_EnvironmentError (C var) - single: PyExc_IOError (C var) - single: PyExc_WindowsError (C var) +OSError aliases +--------------- -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`!PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ +The following are a compatibility aliases to :c:data:`PyExc_OSError`. .. versionchanged:: 3.3 These aliases used to be separate exception types. +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * Notes + * * .. c:var:: PyObject *PyExc_EnvironmentError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_IOError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_WindowsError + * :exc:`OSError` + * [win]_ + Notes: -.. [1] - This is a base class for other standard exceptions. +.. [win] + :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that + uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. .. _standardwarningcategories: -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning (C var) - single: PyExc_BytesWarning (C var) - single: PyExc_DeprecationWarning (C var) - single: PyExc_EncodingWarning (C var) - single: PyExc_FutureWarning (C var) - single: PyExc_ImportWarning (C var) - single: PyExc_PendingDeprecationWarning (C var) - single: PyExc_ResourceWarning (C var) - single: PyExc_RuntimeWarning (C var) - single: PyExc_SyntaxWarning (C var) - single: PyExc_UnicodeWarning (C var) - single: PyExc_UserWarning (C var) - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EncodingWarning` | :exc:`EncodingWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ +Warning types +------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_Warning + * :exc:`Warning` + * * .. c:var:: PyObject *PyExc_BytesWarning + * :exc:`BytesWarning` + * * .. c:var:: PyObject *PyExc_DeprecationWarning + * :exc:`DeprecationWarning` + * * .. c:var:: PyObject *PyExc_EncodingWarning + * :exc:`EncodingWarning` + * * .. c:var:: PyObject *PyExc_FutureWarning + * :exc:`FutureWarning` + * * .. c:var:: PyObject *PyExc_ImportWarning + * :exc:`ImportWarning` + * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning + * :exc:`PendingDeprecationWarning` + * * .. c:var:: PyObject *PyExc_ResourceWarning + * :exc:`ResourceWarning` + * * .. c:var:: PyObject *PyExc_RuntimeWarning + * :exc:`RuntimeWarning` + * * .. c:var:: PyObject *PyExc_SyntaxWarning + * :exc:`SyntaxWarning` + * * .. c:var:: PyObject *PyExc_UnicodeWarning + * :exc:`UnicodeWarning` + * * .. c:var:: PyObject *PyExc_UserWarning + * :exc:`UserWarning` .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. .. versionadded:: 3.10 :c:data:`PyExc_EncodingWarning`. - -Notes: - -.. [3] - This is a base class for other standard warning categories. diff --git a/Doc/conf.py b/Doc/conf.py index 8b2a8f20fcc558..c1ed94d7b46ec2 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -233,75 +233,6 @@ # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BaseExceptionGroup'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_PythonFinalizationError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_EncodingWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), # Undocumented public C macros ('c:macro', 'Py_BUILD_ASSERT'), ('c:macro', 'Py_BUILD_ASSERT_EXPR'), From da8bcfd949885a9a057038ef0095f442b05ee29a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 15:13:34 +0200 Subject: [PATCH 610/713] [3.14] gh-136394: Fix race condition in test_zstd (GH-136432) (GH-136506) gh-136394: Fix race condition in test_zstd (GH-136432) (cherry picked from commit f519918ec6c125715d4efc9713ba80e83346e466) Co-authored-by: Rogdham <3994389+Rogdham@users.noreply.github.com> --- Lib/test/test_zstd.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index e83caaf4c07b13..cf618534add387 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -2674,8 +2674,12 @@ def test_compress_locking(self): input = b'a'* (16*_1K) num_threads = 8 + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + comp = ZstdCompressor() - parts = [] + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] for _ in range(num_threads): res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2684,7 +2688,7 @@ def test_compress_locking(self): expected = b''.join(parts) + rest1 comp = ZstdCompressor() - output = [] + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] def run_method(method, input_data, output_data): res = method(input_data, ZstdCompressor.FLUSH_BLOCK) if res: From 95baa28d9ffaf388fd426ef4474960ccb73c81cf Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 16:52:18 +0200 Subject: [PATCH 611/713] [3.14] gh-132661: Add default value (of `""`) for `Interpolation.expression` (GH-136441) (#136511) Co-authored-by: Dave Peck --- Lib/test/test_string/_support.py | 44 ++++++++++++------- Lib/test/test_string/test_templatelib.py | 13 ++++++ ...-07-08-23-53-51.gh-issue-132661.B84iYt.rst | 1 + Objects/clinic/interpolationobject.c.h | 23 ++++++---- Objects/interpolationobject.c | 4 +- 5 files changed, 58 insertions(+), 27 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py index eaa3354a559246..cfead782b7d4c1 100644 --- a/Lib/test/test_string/_support.py +++ b/Lib/test/test_string/_support.py @@ -3,33 +3,45 @@ class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + def assertTStringEqual(self, t, strings, interpolations): """Test template string literal equality. The *strings* argument must be a tuple of strings equal to *t.strings*. The *interpolations* argument must be a sequence of tuples which are - compared against *t.interpolations*. Each tuple consists of - (value, expression, conversion, format_spec), though the final two - items may be omitted, and are assumed to be None and '' respectively. + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. """ self.assertEqual(t.strings, strings) self.assertEqual(len(t.interpolations), len(interpolations)) for i, exp in zip(t.interpolations, interpolations, strict=True): - if len(exp) == 4: - actual = (i.value, i.expression, i.conversion, i.format_spec) - self.assertEqual(actual, exp) - continue - - if len(exp) == 3: - self.assertEqual((i.value, i.expression, i.conversion), exp) - self.assertEqual(i.format_spec, '') - continue - - self.assertEqual((i.value, i.expression), exp) - self.assertEqual(i.format_spec, '') - self.assertIsNone(i.conversion) + self.assertInterpolationEqual(i, exp) def convert(value, conversion): diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index 85fcff486d6616..adaf590e64dad6 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -45,6 +45,19 @@ def test_basic_creation(self): self.assertEqual(len(t.interpolations), 0) self.assertEqual(fstring(t), 'Hello,\nworld') + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + def test_creation_interleaving(self): # Should add strings on either side t = Template(Interpolation('Maria', 'name', None, '')) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst new file mode 100644 index 00000000000000..9930413b53c150 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst @@ -0,0 +1 @@ +``Interpolation.expression`` now has a default, the empty string. diff --git a/Objects/clinic/interpolationobject.c.h b/Objects/clinic/interpolationobject.c.h index 7a94dabafc92f2..2030e17e49e47a 100644 --- a/Objects/clinic/interpolationobject.c.h +++ b/Objects/clinic/interpolationobject.c.h @@ -47,26 +47,31 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *value; - PyObject *expression; + PyObject *expression = &_Py_STR(empty); PyObject *conversion = Py_None; PyObject *format_spec = &_Py_STR(empty); fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } value = fastargs[0]; - if (!PyUnicode_Check(fastargs[1])) { - _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); - goto exit; - } - expression = fastargs[1]; if (!noptargs) { goto skip_optional_pos; } + if (fastargs[1]) { + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); + goto exit; + } + expression = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } if (fastargs[2]) { if (!_conversion_converter(fastargs[2], &conversion)) { goto exit; @@ -86,4 +91,4 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=599742a5ccd6f060 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2391391e2d7708c0 input=a9049054013a1b77]*/ diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index a5d407a7b0e296..b58adb693f0cae 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -54,7 +54,7 @@ typedef struct { Interpolation.__new__ as interpolation_new value: object - expression: object(subclass_of='&PyUnicode_Type') + expression: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" conversion: object(converter='_conversion_converter') = None format_spec: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" [clinic start generated code]*/ @@ -63,7 +63,7 @@ static PyObject * interpolation_new_impl(PyTypeObject *type, PyObject *value, PyObject *expression, PyObject *conversion, PyObject *format_spec) -/*[clinic end generated code: output=6488e288765bc1a9 input=d91711024068528c]*/ +/*[clinic end generated code: output=6488e288765bc1a9 input=fc5c285c1dd23d36]*/ { interpolationobject *self = PyObject_GC_New(interpolationobject, type); if (!self) { From 326fe0df66afbe94475e5609adeb1ea7a4956899 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 17:41:31 +0200 Subject: [PATCH 612/713] [3.14] gh-82663: Clarify `codecs.iterdecode/encode` docs (GH-136497) (#136513) gh-82663: Clarify `codecs.iterdecode/encode` docs (GH-136497) Closes GH-82663 (cherry picked from commit 4b41b2043b110a38616ff86ddb3f065ae7f15c3e) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/codecs.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 0e84f18dd4d5d5..37bd913b765d21 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -243,8 +243,8 @@ wider range of codecs when working with binary files: .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`str` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental encoder. This function requires that the codec accept text :class:`str` objects @@ -255,8 +255,8 @@ wider range of codecs when working with binary files: .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`bytes` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental decoder. This function requires that the codec accept :class:`bytes` objects From 183b020cb5960e17b87c34a98ec02fcf2b840578 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 10 Jul 2025 18:09:45 +0200 Subject: [PATCH 613/713] [3.14] gh-52876: Implement missing parameter in `codecs.StreamReaderWriter` functions (GH-136498) (#136514) gh-52876: Implement missing parameter in `codecs.StreamReaderWriter` functions (GH-136498) Closes GH-52876 (cherry picked from commit 35e2c359703e076256c1249b74b87043972e04d6) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Lib/codecs.py | 10 +++++----- .../2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst | 3 +++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst diff --git a/Lib/codecs.py b/Lib/codecs.py index fc38e922257644..e4a8010aba90a5 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -618,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -709,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst b/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst new file mode 100644 index 00000000000000..a835306868d3dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst @@ -0,0 +1,3 @@ +Add missing ``keepends`` (default ``True``) parameter to +:meth:`!codecs.StreamReaderWriter.readline` and +:meth:`!codecs.StreamReaderWriter.readlines`. From ec15251af5c937516e4da1f56714c009997c8d5e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 11:43:52 +0200 Subject: [PATCH 614/713] [3.14] gh-101100: Fix sphinx warnings in `library/email.parser.rst` (GH-136475) (#136532) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/library/email.parser.rst | 10 +++++----- Doc/tools/.nitignore | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 439b5c8f34b65a..90796370ebb407 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -48,8 +48,8 @@ methods. FeedParser API ^^^^^^^^^^^^^^ -The :class:`BytesFeedParser`, imported from the :mod:`email.feedparser` module, -provides an API that is conducive to incremental parsing of email messages, +The :class:`BytesFeedParser`, imported from the :mod:`email.parser.FeedParser` +module, provides an API that is conducive to incremental parsing of email messages, such as would be necessary when reading the text of an email message from a source that can block (such as a socket). The :class:`BytesFeedParser` can of course be used to parse an email message fully contained in a :term:`bytes-like @@ -116,7 +116,7 @@ Here is the API for the :class:`BytesFeedParser`: Works like :class:`BytesFeedParser` except that the input to the :meth:`~BytesFeedParser.feed` method must be a string. This is of limited utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is + contain only ASCII text or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, no binary attachments. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -155,11 +155,11 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.TextIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) + (or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, :rfc:`6532`) style headers and header continuation lines, optionally preceded by an envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 4f5396857f3024..897b404b8d6bb3 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -15,7 +15,6 @@ Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst -Doc/library/email.parser.rst Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst From fdad31924c57f30056c69dca382e9159e90b837f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 12:56:34 +0200 Subject: [PATCH 615/713] [3.14] gh-76637: Note that `undefined` Codec is for testing (GH-136531) (#136536) gh-76637: Note that `undefined` Codec is for testing (GH-136531) Closes GH-76637 (cherry picked from commit 975b57d945c84000949f241ded8f44413ecc6217) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/codecs.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 37bd913b765d21..c5dae7c8e8fd04 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1395,7 +1395,11 @@ encodings. | | | It is used in the Python | | | | pickle protocol. | +--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | +| undefined | | This Codec should only | +| | | be used for testing | +| | | purposes. | +| | | | +| | | Raise an exception for | | | | all conversions, even | | | | empty strings. The error | | | | handler is ignored. | From a464c4e2e844db0dc09bad4b29d2e765f8fbe9ab Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 14:57:52 +0200 Subject: [PATCH 616/713] [3.14] gh-136434: Fix docs generation of `UnboundItem` in subinterpreters (GH-136435) (#136540) gh-136434: Fix docs generation of `UnboundItem` in subinterpreters (GH-136435) (cherry picked from commit 3343fce05acb29a772599ce586abd43edf40bae6) Co-authored-by: sobolevn --- Lib/concurrent/interpreters/_crossinterp.py | 19 ++++++++++++------- ...-07-08-20-58-01.gh-issue-136434.uuJsjS.rst | 2 ++ 2 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst diff --git a/Lib/concurrent/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py index f47eb693ac861c..a5f46b20fbb4c5 100644 --- a/Lib/concurrent/interpreters/_crossinterp.py +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -40,16 +40,21 @@ class UnboundItem: @classonly def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) + doc = cls.__doc__ + if doc: + doc = doc.replace( + 'cross-interpreter container', kind, + ).replace( + 'cross-interpreter', kind, + ) subclass = type( f'Unbound{kind.capitalize()}Item', (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), + { + "_MODULE": module, + "_NAME": name, + "__doc__": doc, + }, ) return object.__new__(subclass) diff --git a/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst b/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst new file mode 100644 index 00000000000000..951f57100b650c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst @@ -0,0 +1,2 @@ +Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` +when running with :option:`-OO`. From 5535482d2a53ae18859095f569ac95afed3cb095 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 16:06:19 +0200 Subject: [PATCH 617/713] [3.14] gh-136541: Fix several problems of perf trampolines in x86_64 and aarch64 (GH-136500) (#136544) gh-136541: Fix several problems of perf trampolines in x86_64 and aarch64 (GH-136500) This commit fixes the following problems: * The x86_64 trampolines are not preserving frame pointers * The hardcoded offsets to the code segment from the FDE only worked properly for x64_64 * The CIE data was not following conventions of aarch64 * The eh_frame for aarch64 was not fully correct (cherry picked from commit 236f733d8ffb3d587e1167fa0a0248c24512e7fd) Co-authored-by: Pablo Galindo Salgado --- ...-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst | 3 + Python/asm_trampoline.S | 7 +- Python/perf_jit_trampoline.c | 162 ++++++++++++++---- Python/perf_trampoline.c | 16 +- 4 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst new file mode 100644 index 00000000000000..af9b94ad0613d7 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst @@ -0,0 +1,3 @@ +Fix some issues with the perf trampolines on x86-64 and aarch64. The +trampolines were not being generated correctly for some cases, which could +lead to the perf integration not working correctly. Patch by Pablo Galindo. diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 616752459ba4d9..a14e68c0e81932 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -12,9 +12,10 @@ _Py_trampoline_func_start: #if defined(__CET__) && (__CET__ & 1) endbr64 #endif - sub $8, %rsp - call *%rcx - add $8, %rsp + push %rbp + mov %rsp, %rbp + call *%rcx + pop %rbp ret #endif // __x86_64__ #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 2ca18c23593547..fafe393065bf6b 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -97,10 +97,9 @@ * /tmp/jitted-PID-0.so: [headers][.text][unwind_info][padding] * /tmp/jitted-PID-1.so: [headers][.text][unwind_info][padding] * - * The padding size (0x100) is chosen to accommodate typical unwind info sizes - * while maintaining 16-byte alignment requirements. + * The padding size is now calculated automatically during initialization + * based on the actual unwind information requirements. */ -#define PERF_JIT_CODE_PADDING 0x100 /* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api @@ -401,10 +400,12 @@ enum { DWRF_CFA_nop = 0x0, // No operation DWRF_CFA_offset_extended = 0x5, // Extended offset instruction DWRF_CFA_def_cfa = 0xc, // Define CFA rule + DWRF_CFA_def_cfa_register = 0xd, // Define CFA register DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset DWRF_CFA_advance_loc = 0x40, // Advance location counter - DWRF_CFA_offset = 0x80 // Simple offset instruction + DWRF_CFA_offset = 0x80, // Simple offset instruction + DWRF_CFA_restore = 0xc0 // Restore register }; /* DWARF Exception Handling pointer encodings */ @@ -519,6 +520,7 @@ typedef struct ELFObjectContext { uint8_t* p; // Current write position in buffer uint8_t* startp; // Start of buffer (for offset calculations) uint8_t* eh_frame_p; // Start of EH frame data (for relative offsets) + uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) uint32_t code_size; // Size of the code being described } ELFObjectContext; @@ -643,6 +645,8 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { // DWARF EH FRAME GENERATION // ============================================================================= +static void elf_init_ehframe(ELFObjectContext* ctx); + /* * Initialize DWARF .eh_frame section for a code region * @@ -657,6 +661,23 @@ static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { * Args: * ctx: ELF object context containing code size and buffer pointers */ +static size_t calculate_eh_frame_size(void) { + /* Calculate the EH frame size for the trampoline function */ + extern void *_Py_trampoline_func_start; + extern void *_Py_trampoline_func_end; + + size_t code_size = (char*)&_Py_trampoline_func_end - (char*)&_Py_trampoline_func_start; + + ELFObjectContext ctx; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; + + elf_init_ehframe(&ctx); + return ctx.p - ctx.startp; +} + static void elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; uint8_t* framep = p; // Remember start of frame data @@ -784,7 +805,7 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * * DWRF_SECTION(FDE, * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) - * DWRF_U32(-0x30); // Initial PC-relative location of the code + * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) * DWRF_U32(ctx->code_size); // Code range covered by this FDE * DWRF_U8(0); // Augmentation data length (none) * @@ -830,19 +851,31 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { DWRF_U32(0); // CIE ID (0 indicates this is a CIE) DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) - DWRF_UV(1); // Code alignment factor +#ifdef __x86_64__ + DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) +#endif DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) DWRF_U8(DWRF_REG_RA); // Return address register number DWRF_UV(1); // Augmentation data length DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // FDE pointer encoding /* Initial CFI instructions - describe default calling convention */ +#ifdef __x86_64__ + /* x86_64 initial CFI state */ DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) DWRF_UV(DWRF_REG_SP); // CFA = SP register DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved DWRF_UV(1); // At offset 1 from CFA - +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) + // No initial register saves in AArch64 CIE +#endif DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary ) @@ -853,11 +886,15 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * * The FDE describes unwinding information specific to this function. * It references the CIE and provides function-specific CFI instructions. + * + * The PC-relative offset is calculated after the entire EH frame is built + * to ensure accurate positioning relative to the synthesized DSO layout. */ DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) - DWRF_U32(-0x30); // Machine code offset relative to .text - DWRF_U32(ctx->code_size); // Address range covered by this FDE (code lenght) + ctx->fde_p = p; // Remember where PC offset field is located for later calculation + DWRF_U32(0); // Placeholder for PC-relative offset (calculated at end of elf_init_ehframe) + DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) DWRF_U8(0); // Augmentation data length (none) /* @@ -868,32 +905,36 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { * conventions and register usage patterns. */ #ifdef __x86_64__ - /* x86_64 calling convention unwinding rules */ + /* x86_64 calling convention unwinding rules with frame pointer */ # if defined(__CET__) && (__CET__ & 1) - DWRF_U8(DWRF_CFA_advance_loc | 8); // Advance location by 8 bytes when CET protection is enabled -# else - DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance location by 4 bytes + DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) # endif - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) + DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 DWRF_UV(16); // New offset: SP + 16 - DWRF_U8(DWRF_CFA_advance_loc | 6); // Advance location by 6 bytes - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset + DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 + DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 + DWRF_UV(DWRF_REG_BP); // Use base pointer register + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 + DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 + DWRF_UV(DWRF_REG_SP); // Use stack pointer register DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) /* AArch64 calling convention unwinding rules */ - DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 instruction (stp x29, x30) - DWRF_U8(DWRF_CFA_def_cfa_offset); // Redefine CFA offset - DWRF_UV(16); // CFA = SP + 16 (stack pointer after push) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Frame pointer (x29) saved - DWRF_UV(2); // At offset 2 from CFA (2 * 8 = 16 bytes) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Link register (x30) saved - DWRF_UV(1); // At offset 1 from CFA (1 * 8 = 8 bytes) - DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (mov x16, x3; mov x29, sp; ldp...) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore frame pointer (x29) - DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore link register (x30) - DWRF_U8(DWRF_CFA_def_cfa_offset); // Final CFA adjustment - DWRF_UV(0); // CFA = SP + 0 (stack restored) - + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + DWRF_UV(16); // Stack pointer moved by 16 bytes + DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved + DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) + DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved + DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) + DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) + DWRF_UV(0); // Back to original stack position #else # error "Unsupported target architecture" #endif @@ -902,6 +943,58 @@ static void elf_init_ehframe(ELFObjectContext* ctx) { ) ctx->p = p; // Update context pointer to end of generated data + + /* Calculate and update the PC-relative offset in the FDE + * + * When perf processes the jitdump, it creates a synthesized DSO with this layout: + * + * Synthesized DSO Memory Layout: + * ┌─────────────────────────────────────────────────────────────┐ < code_start + * │ Code Section │ + * │ (round_up(code_size, 8) bytes) │ + * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data + * │ EH Frame Data │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ CIE data │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ FDE Header: │ │ + * │ │ - CIE offset (4 bytes) │ │ + * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start + * │ │ - address range (4 bytes) │ │ (this specific field) + * │ │ CFI Instructions... │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * ├─────────────────────────────────────────────────────────────┤ < reference_point + * │ EhFrameHeader │ + * │ (navigation metadata) │ + * └─────────────────────────────────────────────────────────────┘ + * + * The PC offset field in the FDE must contain the distance from itself to code_start: + * + * distance = code_start - fde_pc_field + * + * Where: + * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame + * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) + * + * Therefore: + * distance = code_start_location - fde_pc_field_location + * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) + * = -rounded_code_size - fde_offset_in_frame + * = -(round_up(code_size, 8) + fde_offset_in_frame) + * + * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field, + * + */ + if (ctx->fde_p != NULL) { + int32_t fde_offset_in_frame = (ctx->fde_p - ctx->startp); + int32_t rounded_code_size = round_up(ctx->code_size, 8); + int32_t pc_relative_offset = -(rounded_code_size + fde_offset_in_frame); + + + // Update the PC-relative offset in the FDE + *(int32_t*)ctx->fde_p = pc_relative_offset; + } } // ============================================================================= @@ -1002,8 +1095,10 @@ static void* perf_map_jit_init(void) { /* Initialize code ID counter */ perf_jit_map_state.code_id = 0; - /* Configure trampoline API with padding information */ - trampoline_api.code_padding = PERF_JIT_CODE_PADDING; + /* Calculate padding size based on actual unwind info requirements */ + size_t eh_frame_size = calculate_eh_frame_size(); + size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + trampoline_api.code_padding = round_up(unwind_data_size, 16); return &perf_jit_map_state; } @@ -1092,6 +1187,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) ctx.code_size = code_size; ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; // Initialize to NULL, will be set when FDE is written /* Generate EH frame (Exception Handling frame) data */ elf_init_ehframe(&ctx); @@ -1110,7 +1206,7 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; /* Verify we don't exceed our padding budget */ - assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding); ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); ev2.mapped_size = round_up(ev2.unwind_data_size, 16); // 16-byte alignment @@ -1262,4 +1358,4 @@ _PyPerf_Callbacks _Py_perfmap_jit_callbacks = { &perf_map_jit_fini, // Cleanup function }; -#endif /* PY_HAVE_PERF_TRAMPOLINE */ \ No newline at end of file +#endif /* PY_HAVE_PERF_TRAMPOLINE */ diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 996e54b82b61e8..4690e7b718f3c8 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -162,6 +162,8 @@ static void invalidate_icache(char* begin, char*end) { } #endif +#define CODE_ALIGNMENT 32 + /* The function pointer is passed as last argument. The other three arguments * are passed in the same order as the function requires. This results in * shorter, more efficient ASM code for trampoline. @@ -291,7 +293,9 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; - size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); + size_t unaligned_size = code_size + trampoline_api.code_padding; + size_t chunk_size = round_up(unaligned_size, CODE_ALIGNMENT); + assert(chunk_size % CODE_ALIGNMENT == 0); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -356,7 +360,9 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, + CODE_ALIGNMENT); + assert(total_code_size % CODE_ALIGNMENT == 0); code_arena->size_left -= total_code_size; code_arena->current_addr += total_code_size; return trampoline; @@ -489,9 +495,6 @@ _PyPerfTrampoline_Init(int activate) } else { _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); - if (new_code_arena() < 0) { - return -1; - } extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { return -1; @@ -499,6 +502,9 @@ _PyPerfTrampoline_Init(int activate) if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { trampoline_api.state = trampoline_api.init_state(); } + if (new_code_arena() < 0) { + return -1; + } perf_status = PERF_STATUS_OK; } #endif From e03db7317f57759c63194b88a5ca886ff9829cf7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 16:20:05 +0200 Subject: [PATCH 618/713] [3.14] gh-136517: Print uncollectable objects if DEBUG_UNCOLLECTABLE mode was set (GH-136518) (#136522) gh-136517: Print uncollectable objects if DEBUG_UNCOLLECTABLE mode was set (GH-136518) (cherry picked from commit c560df9658f1a24edea995fe6f9c84c55b37cfb3) Co-authored-by: Sergey Miryanov --- Lib/test/test_gc.py | 5 +++++ .../2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst | 2 ++ Python/gc.c | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index b4cbfb6d774080..d9e9622992f543 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -726,6 +726,9 @@ def run_command(code): self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " b"shutdown; use", stderr) self.assertNotIn(b"", stderr) + one_line_re = b"gc: uncollectable " + expected_re = one_line_re + b"\r?\n" + one_line_re + self.assertNotRegex(stderr, expected_re) # With DEBUG_UNCOLLECTABLE, the garbage list gets printed stderr = run_command(code % "gc.DEBUG_UNCOLLECTABLE") self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " @@ -733,6 +736,8 @@ def run_command(code): self.assertTrue( (b"[, ]" in stderr) or (b"[, ]" in stderr), stderr) + # we expect two lines with uncollectable objects + self.assertRegex(stderr, expected_re) # With DEBUG_SAVEALL, no additional message should get printed # (because gc.garbage also contains normally reclaimable cyclic # references, and its elements get printed at runtime anyway). diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst new file mode 100644 index 00000000000000..bf26c4eb0e4c74 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst @@ -0,0 +1,2 @@ +Fixed a typo that prevented printing of uncollectable objects when the +:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. diff --git a/Python/gc.c b/Python/gc.c index 7b0e6d6e803504..41854265361b24 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -1763,7 +1763,7 @@ gc_collect_region(PyThreadState *tstate, Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } stats->uncollectable = n; From 3b28b402a52d90c47293d20c8a2cef888b700005 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 18:25:58 +0200 Subject: [PATCH 619/713] [3.14] gh-130478: fix HACL* build for macOS Silicon (GH-134188) (#135009) gh-130478: fix HACL* build for macOS Silicon (GH-134188) (cherry picked from commit ac7511062bf8e16ad489b17990d99abd3b4351f5) Co-authored-by: Sam Ng Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- configure | 12 ++++++++++-- configure.ac | 13 +++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/configure b/configure index d32ea2dabc2be7..a62f1ae7cac52d 100755 --- a/configure +++ b/configure @@ -32582,6 +32582,14 @@ LIBHACL_CFLAGS="${LIBHACL_FLAG_I} ${LIBHACL_FLAG_D} \$(PY_STDMODULE_CFLAGS) \$(C LIBHACL_LDFLAGS= # for now, no specific linker flags are needed +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -32635,7 +32643,7 @@ printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC128 1" >>confdefs.h # isn't great, so it's disabled on ARM64. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 printf %s "checking for HACL* SIMD128 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32712,7 +32720,7 @@ printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC256 1" >>confdefs.h # wrapped implementation if we're building for universal2. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 printf %s "checking for HACL* SIMD256 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } diff --git a/configure.ac b/configure.ac index ab688d93990a2d..d644c0ca772cd5 100644 --- a/configure.ac +++ b/configure.ac @@ -8008,6 +8008,15 @@ AC_SUBST([LIBHACL_CFLAGS]) LIBHACL_LDFLAGS= # for now, no specific linker flags are needed AC_SUBST([LIBHACL_LDFLAGS]) +dnl Check if universal2 HACL* implementation should be used. +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -8025,7 +8034,7 @@ then # available on x86_64. However, performance of the HACL SIMD128 implementation # isn't great, so it's disabled on ARM64. AC_MSG_CHECKING([for HACL* SIMD128 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -8058,7 +8067,7 @@ then # implementation requires symbols that aren't available on ARM64. Use a # wrapped implementation if we're building for universal2. AC_MSG_CHECKING([for HACL* SIMD256 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] AC_MSG_RESULT([universal2]) else From e4e86a1b87eaa6c327632ef282b48b9f15bc5536 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:03:11 +0200 Subject: [PATCH 620/713] [3.14] gh-130160: use `.. program::` directive for documenting `venv` CLI (GH-130699) (#136550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-130160: use `.. program::` directive for documenting `venv` CLI (GH-130699) (cherry picked from commit fb9f933b8eda6cdc1336582dc8709b759ced91af) Co-authored-by: Kanishk Pachauri Co-authored-by: Semyon Moroz Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/venv.rst | 70 +++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index bed799aedfdfb1..f16e24eac08343 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -105,36 +105,52 @@ The command, if run with ``-h``, will show the available options:: Creates virtual Python environments in one or more target directories. - positional arguments: - ENV_DIR A directory to create the environment in. - - options: - -h, --help show this help message and exit - --system-site-packages - Give the virtual environment access to the system - site-packages dir. - --symlinks Try to use symlinks rather than copies, when - symlinks are not the default for the platform. - --copies Try to use copies rather than symlinks, even when - symlinks are the default for the platform. - --clear Delete the contents of the environment directory - if it already exists, before environment creation. - --upgrade Upgrade the environment directory to use this - version of Python, assuming Python has been - upgraded in-place. - --without-pip Skips installing or upgrading pip in the virtual - environment (pip is bootstrapped by default) - --prompt PROMPT Provides an alternative prompt prefix for this - environment. - --upgrade-deps Upgrade core dependencies (pip) to the latest - version in PyPI - --without-scm-ignore-files - Skips adding SCM ignore files to the environment - directory (Git is supported by default). - Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. +.. _venv-cli: +.. program:: venv + +.. option:: ENV_DIR + + A required argument specifying the directory to create the environment in. + +.. option:: --system-site-packages + + Give the virtual environment access to the system site-packages directory. + +.. option:: --symlinks + + Try to use symlinks rather than copies, when symlinks are not the default for the platform. + +.. option:: --copies + + Try to use copies rather than symlinks, even when symlinks are the default for the platform. + +.. option:: --clear + + Delete the contents of the environment directory if it already exists, before environment creation. + +.. option:: --upgrade + + Upgrade the environment directory to use this version of Python, assuming Python has been upgraded in-place. + +.. option:: --without-pip + + Skips installing or upgrading pip in the virtual environment (pip is bootstrapped by default). + +.. option:: --prompt + + Provides an alternative prompt prefix for this environment. + +.. option:: --upgrade-deps + + Upgrade core dependencies (pip) to the latest version in PyPI. + +.. option:: --without-scm-ignore-files + + Skips adding SCM ignore files to the environment directory (Git is supported by default). + .. versionchanged:: 3.4 Installs pip by default, added the ``--without-pip`` and ``--copies`` From ac807cb3a0e2c8b119a19e3622de3a9c4884652d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Fri, 11 Jul 2025 19:09:22 +0200 Subject: [PATCH 621/713] [3.14] gh-101100: Fix sphinx warnings in Doc/library/functools.rst (GH-136424) (GH-136552) Add index entries and anchors for cache_info, cache_clear and register. (cherry picked from commit 252e2f710ea376a38c4545dd758e03d331c1eaad) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/library/functools.rst | 33 +++++++++++++++++++++------------ Doc/tools/.nitignore | 1 - 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3e75621be6dad3..beec9b942afc0f 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -199,12 +199,18 @@ The :mod:`functools` module defines the following functions: and *typed*. This is for information purposes only. Mutating the values has no effect. + .. method:: lru_cache.cache_info() + :no-typesetting: + To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` + parameter, the wrapped function is instrumented with a :func:`!cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, *maxsize* and *currsize*. - The decorator also provides a :func:`cache_clear` function for clearing or + .. method:: lru_cache.cache_clear() + :no-typesetting: + + The decorator also provides a :func:`!cache_clear` function for clearing or invalidating the cache. The original underlying function is accessible through the @@ -284,9 +290,9 @@ The :mod:`functools` module defines the following functions: class decorator supplies the rest. This simplifies the effort involved in specifying all of the possible rich comparison operations: - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. + The class must define one of :meth:`~object.__lt__`, :meth:`~object.__le__`, + :meth:`~object.__gt__`, or :meth:`~object.__ge__`. + In addition, the class should supply an :meth:`~object.__eq__` method. For example:: @@ -418,7 +424,7 @@ The :mod:`functools` module defines the following functions: like normal functions, are handled as descriptors). When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or + :func:`classmethod`, :func:`staticmethod`, :func:`~abc.abstractmethod` or another instance of :class:`partialmethod`), calls to ``__get__`` are delegated to the underlying descriptor, and an appropriate :ref:`partial object` returned as the result. @@ -499,7 +505,10 @@ The :mod:`functools` module defines the following functions: ... print("Let me just say,", end=" ") ... print(arg) - To add overloaded implementations to the function, use the :func:`register` + .. method:: singledispatch.register() + :no-typesetting: + + To add overloaded implementations to the function, use the :func:`!register` attribute of the generic function, which can be used as a decorator. For functions annotated with types, the decorator will infer the type of the first argument automatically:: @@ -565,14 +574,14 @@ The :mod:`functools` module defines the following functions: runtime impact. To enable registering :term:`lambdas` and pre-existing functions, - the :func:`register` attribute can also be used in a functional form:: + the :func:`~singledispatch.register` attribute can also be used in a functional form:: >>> def nothing(arg, verbose=False): ... print("Nothing.") ... >>> fun.register(type(None), nothing) - The :func:`register` attribute returns the undecorated function. This + The :func:`~singledispatch.register` attribute returns the undecorated function. This enables decorator stacking, :mod:`pickling`, and the creation of unit tests for each variant independently:: @@ -650,10 +659,10 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 .. versionchanged:: 3.7 - The :func:`register` attribute now supports using type annotations. + The :func:`~singledispatch.register` attribute now supports using type annotations. .. versionchanged:: 3.11 - The :func:`register` attribute now supports + The :func:`~singledispatch.register` attribute now supports :class:`typing.Union` as a type annotation. @@ -783,7 +792,7 @@ The :mod:`functools` module defines the following functions: 'Docstring' Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` + would have been ``'wrapper'``, and the docstring of the original :func:`!example` would have been lost. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 897b404b8d6bb3..1fbb45ecd73801 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -15,7 +15,6 @@ Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst -Doc/library/functools.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst From 33f561d7be78b984f67a7f97ef11d129b2ff4f3a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 14:33:53 +0200 Subject: [PATCH 622/713] [3.14] gh-89083: Add CLI tests for `UUIDv{6,7,8}` (GH-136548) (#136576) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-89083: Add CLI tests for `UUIDv{6,7,8}` (GH-136548) (cherry picked from commit c564847e98db462edfc30a971da061eeb775e475) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_uuid.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 7ddacf07a2c789..0e1a723ce3a151 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1140,6 +1140,23 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -1214,13 +1231,25 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid def check_has_stable_libuuid_extractable_node(self): From c71ecd14188e6c158fee27a9c2ae75bf69f55f52 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 15:56:10 +0200 Subject: [PATCH 623/713] [3.14] gh-134759: fix `UnboundLocalError` in `email.message.Message.get_payload` (GH-136071) (#136579) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-134759: fix `UnboundLocalError` in `email.message.Message.get_payload` (GH-136071) (cherry picked from commit 25335d297b5248922a4c82183bcdf0c0ada8352b) Co-authored-by: Kliment Lamonov Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/email/message.py | 2 ++ Lib/test/test_email/test_message.py | 9 +++++++++ Misc/ACKS | 1 + .../2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst | 2 ++ 4 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst diff --git a/Lib/email/message.py b/Lib/email/message.py index 41fcc2b9778798..36e4b4a9f0b773 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -313,6 +313,8 @@ def get_payload(self, i=None, decode=False): # If it does happen, turn the string into bytes in a way # guaranteed not to fail. bpayload = payload.encode('raw-unicode-escape') + else: + bpayload = payload if cte == 'quoted-printable': return quopri.decodestring(bpayload) elif cte == 'base64': diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 23c39775a8b2e5..b4128f70f18412 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1055,6 +1055,15 @@ def test_get_body_malformed(self): # AttributeError: 'str' object has no attribute 'is_attachment' m.get_body() + def test_get_bytes_payload_with_quoted_printable_encoding(self): + # We use a memoryview to avoid directly changing the private payload + # and to prevent using the dedicated paths for string or bytes objects. + payload = memoryview(b'Some payload') + m = self._make_message() + m.add_header('Content-Transfer-Encoding', 'quoted-printable') + m.set_payload(payload) + self.assertEqual(m.get_payload(decode=True), payload) + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two diff --git a/Misc/ACKS b/Misc/ACKS index f80ae7d8fac461..83bc62726eecc9 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1054,6 +1054,7 @@ Alexander Lakeev David Lam Thomas Lamb Valerie Lambert +Kliment Lamonov Peter Lamut Jean-Baptiste "Jiba" Lamy Ronan Lamy diff --git a/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst new file mode 100644 index 00000000000000..79b85320926954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst @@ -0,0 +1,2 @@ +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` when +the payload to decode is a :class:`bytes` object. Patch by Kliment Lamonov. From d5866a8e8432c15eac65a3059d71bc60d9aa61bf Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 16:38:57 +0200 Subject: [PATCH 624/713] [3.14] gh-91153: prevent a crash in `bytearray.__setitem__(ind, ...)` when `ind.__index__` has side-effects (GH-132379) (#136581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-91153: prevent a crash in `bytearray.__setitem__(ind, ...)` when `ind.__index__` has side-effects (GH-132379) (cherry picked from commit 5e1e21dee35b8e9066692d08033bbbdb562e2c28) Co-authored-by: Bast <52266665+bast0006@users.noreply.github.com> Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_bytes.py | 35 +++++++++++++++++++ ...5-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 1 + Objects/bytearrayobject.c | 8 +++-- 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index bb0f8aa99da910..2591e7ca6ab0ec 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1899,6 +1899,8 @@ def test_repeat_after_setslice(self): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer # See gh-91153 class Boom: @@ -1916,6 +1918,39 @@ def __index__(self): with self.assertRaises(IndexError): self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + def test_mutating_index_inbounds(self): + # gh-91153 continued + # Ensure buffer is not broken even if length is correct + + class MutatesOnIndex: + def __init__(self): + self.ba = bytearray(0x180) + + def __index__(self): + self.ba.clear() + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks + return 0 + + with self.subTest("skip_bounds_safety"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + class AssortedBytesTest(unittest.TestCase): # diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst new file mode 100644 index 00000000000000..dc2f1e22ba5b05 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst @@ -0,0 +1 @@ +Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b5d5ca9178ebdb..bf30c06af5d8fa 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -709,7 +709,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +746,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +807,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +848,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, From 64df22c19029b1d99eb7993f16bfe9f484a8405e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 18:18:23 +0200 Subject: [PATCH 625/713] [3.14] gh-136549: Fix signature of threading.excepthook() (GH-136559) (GH-136589) (cherry picked from commit be2c3d284ecce67474a260b8c37e2f1e0628a9cf) Co-authored-by: Serhiy Storchaka --- Lib/test/test_inspect/test_inspect.py | 1 + .../next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst | 1 + Modules/_threadmodule.c | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 55942c2823c492..0a8926e00cf8b9 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -5916,6 +5916,7 @@ def test_sysconfig_module_has_signatures(self): def test_threading_module_has_signatures(self): import threading self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) def test_thread_module_has_signatures(self): import _thread diff --git a/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst new file mode 100644 index 00000000000000..f3050ad5d5aa10 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst @@ -0,0 +1 @@ +Fix signature of :func:`threading.excepthook`. diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 73dfb943845095..121b0f803715af 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -2317,7 +2317,7 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +"_excepthook($module, args, /)\n\ --\n\ \n\ Handle uncaught Thread.run() exception."); From d351f29ef08e2f786cdf3400ceabfffb0ae7791d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 12 Jul 2025 20:22:51 +0200 Subject: [PATCH 626/713] [3.14] gh-101100: Fix sphinx warnings in Doc/library/platform.rst (GH-136562) (GH-136597) (cherry picked from commit 47b01da4ccedd9c00fad4325b3e87d7732abeb6d) Co-authored-by: Weilin Du <108666168+LamentXU123@users.noreply.github.com> --- Doc/library/platform.rst | 4 ++-- Doc/tools/.nitignore | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 5c999054323be5..2f176872985632 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -176,8 +176,8 @@ Cross platform :attr:`processor` is resolved late, on demand. Note: the first two attribute names differ from the names presented by - :func:`os.uname`, where they are named :attr:`sysname` and - :attr:`nodename`. + :func:`os.uname`, where they are named :attr:`!sysname` and + :attr:`!nodename`. Entries which cannot be determined are set to ``''``. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 1fbb45ecd73801..510225afab8933 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -26,7 +26,6 @@ Doc/library/multiprocessing.rst Doc/library/optparse.rst Doc/library/os.rst Doc/library/pickletools.rst -Doc/library/platform.rst Doc/library/profile.rst Doc/library/pyexpat.rst Doc/library/resource.rst From f50cd5bec7c507ea1f775ebb30d8d014030c7b94 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 07:42:51 +0200 Subject: [PATCH 627/713] [3.14] gh-134939: Correct `concurrent.interpreters` source code link (GH-136564) (#136605) gh-134939: Correct `concurrent.interpreters` source code link (GH-136564) (cherry picked from commit 42b251bcebd749eceeb62389e413a3be37cff343) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/concurrent.interpreters.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst index 524d505bcf144f..be9d565f8e0d38 100644 --- a/Doc/library/concurrent.interpreters.rst +++ b/Doc/library/concurrent.interpreters.rst @@ -9,7 +9,7 @@ .. versionadded:: 3.14 -**Source code:** :source:`Lib/concurrent/interpreters.py` +**Source code:** :source:`Lib/concurrent/interpreters` -------------- From a1ed132a19ac8a799d1c187527ce16e543e8146f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 08:52:58 +0200 Subject: [PATCH 628/713] [3.14] Docs: Fix and improve the `PyUnstable_Object_EnableDeferredRefcount` documentation (GH-135323) (GH-136610) Docs: Fix and improve the `PyUnstable_Object_EnableDeferredRefcount` documentation (GH-135323) (cherry picked from commit 0d4fd10fbab2767fad3eb27639905c8885b88c89) Co-authored-by: Peter Bierma --- Doc/c-api/object.rst | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0fd159f1eb87f8..241fbd3a4866c7 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -596,12 +596,13 @@ Object Protocol if supported by the runtime. In the :term:`free-threaded ` build, this allows the interpreter to avoid reference count adjustments to *obj*, which may improve multi-threaded performance. The tradeoff is - that *obj* will only be deallocated by the tracing garbage collector. + that *obj* will only be deallocated by the tracing garbage collector, and + not when the interpreter no longer has any references to it. - This function returns ``1`` if deferred reference counting is enabled on *obj* - (including when it was enabled before the call), + This function returns ``1`` if deferred reference counting is enabled on *obj*, and ``0`` if deferred reference counting is not supported or if the hint was - ignored by the runtime. This function is thread-safe, and cannot fail. + ignored by the interpreter, such as when deferred reference counting is already + enabled on *obj*. This function is thread-safe, and cannot fail. This function does nothing on builds with the :term:`GIL` enabled, which do not support deferred reference counting. This also does nothing if *obj* is not @@ -609,7 +610,8 @@ Object Protocol :c:func:`PyObject_GC_IsTracked`). This function is intended to be used soon after *obj* is created, - by the code that creates it. + by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` + slot. .. versionadded:: 3.14 From e1dc08bd9a20783e40c09fc68fd21658e1ec67e7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 08:58:45 +0200 Subject: [PATCH 629/713] [3.14] gh-134833: improve docs for `del s[i:j]` in `Mutable Sequence Types` (GH-134834) (#136608) gh-134833: improve docs for `del s[i:j]` in `Mutable Sequence Types` (GH-134834) (cherry picked from commit 609d5adc7cc241da8fe314a64ddd2c8a883ee8b7) Co-authored-by: Yongzi Li <204532581+Yzi-Li@users.noreply.github.com> --- Doc/library/stdtypes.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 394c302fd354b9..8e688f03eb3a87 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -1223,7 +1223,9 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | | is replaced by the contents of | | | | the iterable *t* | | +------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | +| ``del s[i:j]`` | removes the elements of | | +| | ``s[i:j]`` from the list | | +| | (same as ``s[i:j] = []``) | | +------------------------------+--------------------------------+---------------------+ | ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | | | are replaced by those of *t* | | From 9ee72ab266a15906aa7e714ac601b3186e1ef51c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 11:17:48 +0200 Subject: [PATCH 630/713] [3.14] gh-132346: Docs: Clarify that reference counts aren't stable between versions (GH-132352) (GH-136613) gh-132346: Docs: Clarify that reference counts aren't stable between versions (GH-132352) (cherry picked from commit 3dbe02ccd3eefc48ac9fa14427bb4cdb82d1ebae) Co-authored-by: Peter Bierma --- Doc/glossary.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 705b0a9279c6d4..199a917f9f101e 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -1208,6 +1208,11 @@ Glossary :func:`sys.getrefcount` function to return the reference count for a particular object. + In :term:`CPython`, reference counts are not considered to be stable + or well-defined values; the number of references to an object, and how + that number is affected by Python code, may be different between + versions. + regular package A traditional :term:`package`, such as a directory containing an ``__init__.py`` file. From ed1e0cdc581d328177992e45c74d497c7da22ce2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 15:19:23 +0200 Subject: [PATCH 631/713] [3.14] gh-42237: Link to complete list of codec aliases (GH-136625) (#136626) gh-42237: Link to complete list of codec aliases (GH-136625) Closes GH-42237 (cherry picked from commit a93d9aaf62bb2565e9eec00a2a8d06a91305127b) Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/library/codecs.rst | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index c5dae7c8e8fd04..1cb0e225bca043 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -1065,8 +1065,15 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +case or use a hyphen instead of an underscore are also valid aliases +because they are equivalent when normalized by +:func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid +alias for the ``'utf_8'`` codec. + +.. note:: + + The below table lists the most common aliases, for a complete list + refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. But only codecs listed in the following table are guarantead to exist on From 348e22cf060695fbd57f099b94f0f9067bb49fca Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 15:58:03 +0200 Subject: [PATCH 632/713] [3.14] gh-127971: fix off-by-one read beyond the end of a string during search (GH-132574) (#136628) gh-127971: fix off-by-one read beyond the end of a string during search (GH-132574) (cherry picked from commit 85ec3b3b503ffd5b7e45f8b3fa2cec0c10e4bef0) Co-authored-by: Duane Griffin --- Lib/test/string_tests.py | 9 +++++++++ .../2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst | 1 + Objects/stringlib/fastsearch.h | 8 ++++---- 3 files changed, 14 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 4b82d51b4508ac..1814a55b74ea0c 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -767,6 +767,15 @@ def test_replace(self): self.checkraises(TypeError, 'hello', 'replace', 42, 'h') self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + def test_replacement_on_buffer_boundary(self): + # gh-127971: Check we don't read past the end of the buffer when a + # potential match misses on the last character. + any_3_nonblank_codepoints = '!!!' + seven_codepoints = any_3_nonblank_codepoints + ' ' + any_3_nonblank_codepoints + a = (' ' * 243) + seven_codepoints + (' ' * 7) + b = ' ' * 6 + chr(256) + a.replace(seven_codepoints, b) + def test_replace_uses_two_way_maxcount(self): # Test that maxcount works in _two_way_count in fastsearch.h A, B = "A"*1000, "B"*1000 diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst new file mode 100644 index 00000000000000..ced7a9c9fd3e63 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst @@ -0,0 +1 @@ +Fix off-by-one read beyond the end of a string in string search. diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 05e700b06258f0..b447865c986bef 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -595,7 +595,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, continue; } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -604,7 +604,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } @@ -668,7 +668,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -677,7 +677,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } From ba070b6b07ec0fdd9a0258270e4a111100c0d4db Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 13 Jul 2025 22:52:02 +0200 Subject: [PATCH 633/713] [3.14] gh-135256: Simplify parsing parameters in Argument Clinic (GH-135257) (121914136635) (cherry picked from commit b74fb8e220a50a9580320dfd398a16995b845c69) Co-authored-by: Serhiy Storchaka --- Lib/test/test_clinic.py | 12 ++---- Tools/clinic/libclinic/dsl_parser.py | 64 ++++++---------------------- 2 files changed, 17 insertions(+), 59 deletions(-) diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 05e46a18b0a92a..4b1f5991a39ee8 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -1277,12 +1277,8 @@ def test_base_invalid_syntax(self): os.stat invalid syntax: int = 42 """ - err = dedent(r""" - Function 'stat' has an invalid parameter declaration: - \s+'invalid syntax: int = 42' - """).strip() - with self.assertRaisesRegex(ClinicError, err): - self.parse_function(block) + err = "Function 'stat' has an invalid parameter declaration: 'invalid syntax: int = 42'" + self.expect_failure(block, err, lineno=2) def test_param_default_invalid_syntax(self): block = """ @@ -1290,7 +1286,7 @@ def test_param_default_invalid_syntax(self): os.stat x: int = invalid syntax """ - err = r"Syntax error: 'x = invalid syntax\n'" + err = "Function 'stat' has an invalid parameter declaration:" self.expect_failure(block, err, lineno=2) def test_cloning_nonexistent_function_correctly_fails(self): @@ -2510,7 +2506,7 @@ def test_cannot_specify_pydefault_without_default(self): self.expect_failure(block, err, lineno=1) def test_vararg_cannot_take_default_value(self): - err = "Vararg can't take a default value!" + err = "Function 'fn' has an invalid parameter declaration:" block = """ fn *args: tuple = None diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 282ff64cd33089..eca41531f7c8e9 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -877,43 +877,16 @@ def parse_parameter(self, line: str) -> None: # handle "as" for parameters too c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None + m = re.match(r'(?:\* *)?\w+( +as +(\w+))', line) + if m: + c_name = m[2] + line = line[:m.start(1)] + line[m.end(1):] + try: - ast_input = f"def x({base}): pass" + ast_input = f"def x({line}\n): pass" module = ast.parse(ast_input) except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) + fail(f"Function {self.function.name!r} has an invalid parameter declaration: {line!r}") function = module.body[0] assert isinstance(function, ast.FunctionDef) @@ -922,9 +895,6 @@ def parse_parameter(self, line: str) -> None: if len(function_args.args) > 1: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (**kwargs?): {line!r}") @@ -944,7 +914,7 @@ def parse_parameter(self, line: str) -> None: name = 'varpos_' + name value: object - if not default: + if not function_args.defaults: if is_vararg: value = NULL else: @@ -955,17 +925,13 @@ def parse_parameter(self, line: str) -> None: if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") else: - if is_vararg: - fail("Vararg can't take a default value!") + expr = function_args.defaults[0] + default = ast_input[expr.col_offset: expr.end_col_offset].strip() if self.parameter_state is ParamState.REQUIRED: self.parameter_state = ParamState.OPTIONAL - default = default.strip() bad = False - ast_input = f"x = {default}" try: - module = ast.parse(ast_input) - if 'c_default' not in kwargs: # we can only represent very simple data values in C. # detect whether default is okay, via a denylist @@ -992,13 +958,14 @@ def bad_node(self, node: ast.AST) -> None: visit_Starred = bad_node denylist = DetectBadNodes() - denylist.visit(module) + denylist.visit(expr) bad = denylist.bad else: # if they specify a c_default, we can be more lenient about the default value. # but at least make an attempt at ensuring it's a valid expression. + code = compile(ast.Expression(expr), '', 'eval') try: - value = eval(default) + value = eval(code) except NameError: pass # probably a named constant except Exception as e: @@ -1010,9 +977,6 @@ def bad_node(self, node: ast.AST) -> None: if bad: fail(f"Unsupported expression as default value: {default!r}") - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value # mild hack: explicitly support NULL as a default value c_default: str | None if isinstance(expr, ast.Name) and expr.id == 'NULL': @@ -1064,8 +1028,6 @@ def bad_node(self, node: ast.AST) -> None: else: c_default = py_default - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown c_default = kwargs.get("c_default") From 261818061eb38ac42f9bc35335361c8c55f47b3b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 14 Jul 2025 04:49:36 +0200 Subject: [PATCH 634/713] [3.14] gh-127146: Emscripten: Fix test_open_undecodable_uri by setting `-sTEXTDECODER=2` (GH-136624) (#136631) Removes the JS text decoder fallback and gets rid of the bugs due to the differences in behavior on invalid utf8 strings. See https://github.com/emscripten-core/emscripten/issues/24690. (cherry picked from commit 283b05052338dd735cd4927011afc3735d9c6c7c) Co-authored-by: Hood Chatham --- configure | 1 + configure.ac | 2 ++ 2 files changed, 3 insertions(+) diff --git a/configure b/configure index a62f1ae7cac52d..67a40b841aa506 100755 --- a/configure +++ b/configure @@ -9606,6 +9606,7 @@ fi as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" + as_fn_append LINKFORSHARED " -sTEXTDECODER=2" if test "x$enable_wasm_dynamic_linking" = xyes then : diff --git a/configure.ac b/configure.ac index d644c0ca772cd5..bbd0fa8c1ae8f6 100644 --- a/configure.ac +++ b/configure.ac @@ -2338,6 +2338,8 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) + dnl Avoid bugs in JS fallback string decoding path + AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) From bbbbb2e2d15794bfcaacfdf2bce43ca7032fe50f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 14 Jul 2025 12:19:23 +0200 Subject: [PATCH 635/713] [3.14] Partially revert "gh-101100: Fix sphinx warnings in `library/email.parser.rst` (GH-136475)" (GH-136629) (#136646) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/email.parser.rst | 6 +++--- Doc/tools/.nitignore | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 90796370ebb407..6a70714dc3ee42 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -48,8 +48,8 @@ methods. FeedParser API ^^^^^^^^^^^^^^ -The :class:`BytesFeedParser`, imported from the :mod:`email.parser.FeedParser` -module, provides an API that is conducive to incremental parsing of email messages, +The :class:`BytesFeedParser`, imported from the :mod:`email.feedparser` module, +provides an API that is conducive to incremental parsing of email messages, such as would be necessary when reading the text of an email message from a source that can block (such as a socket). The :class:`BytesFeedParser` can of course be used to parse an email message fully contained in a :term:`bytes-like @@ -155,7 +155,7 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.TextIOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 510225afab8933..31b2e567f11056 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -15,6 +15,7 @@ Doc/extending/extending.rst Doc/library/ast.rst Doc/library/asyncio-extending.rst Doc/library/email.charset.rst +Doc/library/email.parser.rst Doc/library/http.cookiejar.rst Doc/library/http.server.rst Doc/library/importlib.rst From 55eaaab8a4064ef76013a82c16c9991eba30dfeb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:33:33 +0200 Subject: [PATCH 636/713] [3.14] gh-116738: Make grp module thread-safe (GH-135434) (#136658) gh-116738: Make grp module thread-safe (GH-135434) Make grp module methods getgrgid() and getgrnam() thread-safe when the GIL is disabled and getgrgid_r()/getgrnam_r() C APIs are not available. --------- (cherry picked from commit 9363703bd3bf86e363c14a02e3d729caf1e29f44) Co-authored-by: Alper Co-authored-by: Kumar Aditya --- Doc/library/test.rst | 7 ++ Lib/test/support/threading_helper.py | 24 +++++++ Lib/test/test_free_threading/test_grp.py | 35 ++++++++++ Lib/test/test_free_threading/test_heapq.py | 66 +++++++------------ ...-06-12-00-03-34.gh-issue-116738.iBBAdo.rst | 1 + Modules/grpmodule.c | 27 +++++++- Tools/c-analyzer/cpython/ignored.tsv | 1 + 7 files changed, 115 insertions(+), 46 deletions(-) create mode 100644 Lib/test/test_free_threading/test_grp.py create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0aae14c15a6104..9fdb21b8badbbf 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -1384,6 +1384,13 @@ The :mod:`test.support.threading_helper` module provides support for threading t .. versionadded:: 3.8 +.. function:: run_concurrently(worker_func, nthreads, args=(), kwargs={}) + + Run the worker function concurrently in multiple threads. + Re-raises an exception if any thread raises one, after all threads have + finished. + + :mod:`test.support.os_helper` --- Utilities for os tests ======================================================================== diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index afa25a76f63829..3e04c344a0d66f 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -248,3 +248,27 @@ def requires_working_threading(*, module=False): raise unittest.SkipTest(msg) else: return unittest.skipUnless(can_start_thread, msg) + + +def run_concurrently(worker_func, nthreads, args=(), kwargs={}): + """ + Run the worker function concurrently in multiple threads. + """ + barrier = threading.Barrier(nthreads) + + def wrapper_func(*args, **kwargs): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + worker_func(*args, **kwargs) + + with catch_threading_exception() as cm: + workers = [ + threading.Thread(target=wrapper_func, args=args, kwargs=kwargs) + for _ in range(nthreads) + ] + with start_threads(workers): + pass + + # If a worker thread raises an exception, re-raise it. + if cm.exc_value is not None: + raise cm.exc_value diff --git a/Lib/test/test_free_threading/test_grp.py b/Lib/test/test_free_threading/test_grp.py new file mode 100644 index 00000000000000..1a47a9757702be --- /dev/null +++ b/Lib/test/test_free_threading/test_grp.py @@ -0,0 +1,35 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +grp = import_helper.import_module("grp") + +from test import test_grp + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestGrp(unittest.TestCase): + def setUp(self): + self.test_grp = test_grp.GroupDatabaseTestCase() + + def test_racing_test_values(self): + # test_grp.test_values() calls grp.getgrall() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values, nthreads=NTHREADS + ) + + def test_racing_test_values_extended(self): + # test_grp.test_values_extended() calls grp.getgrall(), grp.getgrgid(), + # grp.getgrnam() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values_extended, + nthreads=NTHREADS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py index ee7adfb2b78d83..d771333ffcc9e0 100644 --- a/Lib/test/test_free_threading/test_heapq.py +++ b/Lib/test/test_free_threading/test_heapq.py @@ -3,10 +3,11 @@ import heapq from enum import Enum -from threading import Thread, Barrier, Lock +from threading import Barrier, Lock from random import shuffle, randint from test.support import threading_helper +from test.support.threading_helper import run_concurrently from test import test_heapq @@ -28,8 +29,8 @@ def test_racing_heapify(self): heap = list(range(OBJECT_COUNT)) shuffle(heap) - self.run_concurrently( - worker_func=heapq.heapify, args=(heap,), nthreads=NTHREADS + run_concurrently( + worker_func=heapq.heapify, nthreads=NTHREADS, args=(heap,) ) self.test_heapq.check_invariant(heap) @@ -40,8 +41,8 @@ def heappush_func(heap): for item in reversed(range(OBJECT_COUNT)): heapq.heappush(heap, item) - self.run_concurrently( - worker_func=heappush_func, args=(heap,), nthreads=NTHREADS + run_concurrently( + worker_func=heappush_func, nthreads=NTHREADS, args=(heap,) ) self.test_heapq.check_invariant(heap) @@ -61,10 +62,10 @@ def heappop_func(heap, pop_count): # Each local list should be sorted self.assertTrue(self.is_sorted_ascending(local_list)) - self.run_concurrently( + run_concurrently( worker_func=heappop_func, - args=(heap, per_thread_pop_count), nthreads=NTHREADS, + args=(heap, per_thread_pop_count), ) self.assertEqual(len(heap), 0) @@ -77,10 +78,10 @@ def heappushpop_func(heap, pushpop_items): popped_item = heapq.heappushpop(heap, item) self.assertTrue(popped_item <= item) - self.run_concurrently( + run_concurrently( worker_func=heappushpop_func, - args=(heap, pushpop_items), nthreads=NTHREADS, + args=(heap, pushpop_items), ) self.assertEqual(len(heap), OBJECT_COUNT) self.test_heapq.check_invariant(heap) @@ -93,10 +94,10 @@ def heapreplace_func(heap, replace_items): for item in replace_items: heapq.heapreplace(heap, item) - self.run_concurrently( + run_concurrently( worker_func=heapreplace_func, - args=(heap, replace_items), nthreads=NTHREADS, + args=(heap, replace_items), ) self.assertEqual(len(heap), OBJECT_COUNT) self.test_heapq.check_invariant(heap) @@ -105,8 +106,8 @@ def test_racing_heapify_max(self): max_heap = list(range(OBJECT_COUNT)) shuffle(max_heap) - self.run_concurrently( - worker_func=heapq.heapify_max, args=(max_heap,), nthreads=NTHREADS + run_concurrently( + worker_func=heapq.heapify_max, nthreads=NTHREADS, args=(max_heap,) ) self.test_heapq.check_max_invariant(max_heap) @@ -117,8 +118,8 @@ def heappush_max_func(max_heap): for item in range(OBJECT_COUNT): heapq.heappush_max(max_heap, item) - self.run_concurrently( - worker_func=heappush_max_func, args=(max_heap,), nthreads=NTHREADS + run_concurrently( + worker_func=heappush_max_func, nthreads=NTHREADS, args=(max_heap,) ) self.test_heapq.check_max_invariant(max_heap) @@ -138,10 +139,10 @@ def heappop_max_func(max_heap, pop_count): # Each local list should be sorted self.assertTrue(self.is_sorted_descending(local_list)) - self.run_concurrently( + run_concurrently( worker_func=heappop_max_func, - args=(max_heap, per_thread_pop_count), nthreads=NTHREADS, + args=(max_heap, per_thread_pop_count), ) self.assertEqual(len(max_heap), 0) @@ -154,10 +155,10 @@ def heappushpop_max_func(max_heap, pushpop_items): popped_item = heapq.heappushpop_max(max_heap, item) self.assertTrue(popped_item >= item) - self.run_concurrently( + run_concurrently( worker_func=heappushpop_max_func, - args=(max_heap, pushpop_items), nthreads=NTHREADS, + args=(max_heap, pushpop_items), ) self.assertEqual(len(max_heap), OBJECT_COUNT) self.test_heapq.check_max_invariant(max_heap) @@ -170,10 +171,10 @@ def heapreplace_max_func(max_heap, replace_items): for item in replace_items: heapq.heapreplace_max(max_heap, item) - self.run_concurrently( + run_concurrently( worker_func=heapreplace_max_func, - args=(max_heap, replace_items), nthreads=NTHREADS, + args=(max_heap, replace_items), ) self.assertEqual(len(max_heap), OBJECT_COUNT) self.test_heapq.check_max_invariant(max_heap) @@ -203,7 +204,7 @@ def worker(): except IndexError: pass - self.run_concurrently(worker, (), n_threads * 2) + run_concurrently(worker, n_threads * 2) @staticmethod def is_sorted_ascending(lst): @@ -241,27 +242,6 @@ def create_random_list(a, b, size): """ return [randint(-a, b) for _ in range(size)] - def run_concurrently(self, worker_func, args, nthreads): - """ - Run the worker function concurrently in multiple threads. - """ - barrier = Barrier(nthreads) - - def wrapper_func(*args): - # Wait for all threads to reach this point before proceeding. - barrier.wait() - worker_func(*args) - - with threading_helper.catch_threading_exception() as cm: - workers = ( - Thread(target=wrapper_func, args=args) for _ in range(nthreads) - ) - with threading_helper.start_threads(workers): - pass - - # Worker threads should not raise any exceptions - self.assertIsNone(cm.exc_value) - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst new file mode 100644 index 00000000000000..2a1ed2944d8ddf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst @@ -0,0 +1 @@ +Make functions in :mod:`grp` thread-safe on the :term:`free threaded ` build. diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 29da9936b65504..652958618a2c4c 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -55,6 +55,11 @@ get_grp_state(PyObject *module) static struct PyModuleDef grpmodule; +/* Mutex to protect calls to getgrgid(), getgrnam(), and getgrent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex group_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -168,9 +173,15 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrgid() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrgid.html p = getgrgid(gid); #endif if (p == NULL) { +#ifndef HAVE_GETGRGID_R + PyMutex_Unlock(&group_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -185,6 +196,8 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) retval = mkgrent(module, p); #ifdef HAVE_GETGRGID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&group_db_mutex); #endif return retval; } @@ -249,9 +262,15 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrnam() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrnam.html p = getgrnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -261,6 +280,9 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkgrent(module, p); +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -285,8 +307,7 @@ grp_getgrall_impl(PyObject *module) return NULL; } - static PyMutex getgrall_mutex = {0}; - PyMutex_Lock(&getgrall_mutex); + PyMutex_Lock(&group_db_mutex); setgrent(); struct group *p; @@ -306,7 +327,7 @@ grp_getgrall_impl(PyObject *module) done: endgrent(); - PyMutex_Unlock(&getgrall_mutex); + PyMutex_Unlock(&group_db_mutex); return d; } diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b128abca39fb41..d5d806be42f39d 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -167,6 +167,7 @@ Python/sysmodule.c - _preinit_xoptions - # XXX need race protection? Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/faulthandler.c faulthandler_dump_c_stack reentrant - +Modules/grpmodule.c - group_db_mutex - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - From 088d79e3ee86d31fe69caed1247db7b624f243b5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:36:47 +0200 Subject: [PATCH 637/713] [3.14] gh-136663: fix signatures of PyFloat_Pack/Unpack in docs (GH-136664) (#136666) gh-136663: fix signatures of PyFloat_Pack/Unpack in docs (GH-136664) (cherry picked from commit e4654e0b3e7d802c8fe984cf39a36a42b67de1ad) Co-authored-by: Sergey B Kirpichev --- Doc/c-api/float.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index c5a7653efca26b..489676caa3a16a 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -124,15 +124,15 @@ There are two problems on non-IEEE platforms: * What this does is undefined if *x* is a NaN or infinity. * ``-0.0`` and ``+0.0`` produce the same bytes string. -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack4(double x, char *p, int le) Pack a C double as the IEEE 754 binary32 single precision format. -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack8(double x, char *p, int le) Pack a C double as the IEEE 754 binary64 double precision format. @@ -154,14 +154,14 @@ Return value: The unpacked double. On error, this is ``-1.0`` and Note that on a non-IEEE platform this will refuse to unpack a bytes string that represents a NaN or infinity. -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack4(const char *p, int le) Unpack the IEEE 754 binary32 single precision format as a C double. -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack8(const char *p, int le) Unpack the IEEE 754 binary64 double precision format as a C double. From 115477399bb962193026a3035ab07a1c0cd52f9f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 10:12:45 +0200 Subject: [PATCH 638/713] [3.14] gh-131189: Fix "msvcrt" import warning on Linux when "_ctypes" is not available. (GH-131201) (GH-136668) Fix "msvcrt" import warning on Linux when "_ctypes" is not available. On Linux, compiling without "libffi" causes a "No module named 'msvcrt'" warning when launching PyREPL. (cherry picked from commit f320c951c3220aa6727b581216463e8b3f8bcd6b) Co-authored-by: Dzmitry Plashchynski --- Lib/_pyrepl/readline.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 9560ae779abfea..23b8fa6b9c7625 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -43,10 +43,11 @@ Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" From 2ae0eb1a87a6cb991f9639aedb756841cd5dca69 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:22:24 +0200 Subject: [PATCH 639/713] [3.14] GH-132661: Add ``string.templatelib.convert()`` (GH-135217) (#136671) GH-132661: Add ``string.templatelib.convert()`` (GH-135217) (cherry picked from commit 5b969fd64502a6e2ba6513e2b18beaeae58b8aa1) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Lib/string/templatelib.py | 17 ++++++++++++----- Lib/test/test_string/test_templatelib.py | 22 +++++++++++++++++++++- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py index 14b40e1e36e30b..8164872432ad09 100644 --- a/Lib/string/templatelib.py +++ b/Lib/string/templatelib.py @@ -1,15 +1,22 @@ """Support for template string literals (t-strings).""" -__all__ = [ - "Interpolation", - "Template", -] - t = t"{0}" Template = type(t) Interpolation = type(t.interpolations[0]) del t +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + def _template_unpickle(*args): import itertools diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index adaf590e64dad6..1c86717155fd5a 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -1,7 +1,7 @@ import pickle import unittest from collections.abc import Iterator, Iterable -from string.templatelib import Template, Interpolation +from string.templatelib import Template, Interpolation, convert from test.test_string._support import TStringBaseCase, fstring @@ -169,5 +169,25 @@ def test_exhausted(self): self.assertRaises(StopIteration, next, template_iter) +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Café', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + + if __name__ == '__main__': unittest.main() From 45b72c447cecad5a904507836eaf146a213d81c4 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:09:54 +0200 Subject: [PATCH 640/713] [3.14] gh-72570: mention the incompatibility of XOFs with HMAC (GH-136676) (#136678) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-72570: mention the incompatibility of XOFs with HMAC (GH-136676) (cherry picked from commit a02cf19deed353d1e0e7564468f10aced61c12e8) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/library/hmac.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d6692033b2d4c3..57076c38086c79 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -12,6 +12,9 @@ -------------- This module implements the HMAC algorithm as described by :rfc:`2104`. +The interface allows to use any hash function with a *fixed* digest size. +In particular, extendable output functions such as SHAKE-128 or SHAKE-256 +cannot be used with HMAC. .. function:: new(key, msg=None, digestmod) From c939963b53600062f0978c99559e22ab39135402 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 14:59:38 +0200 Subject: [PATCH 641/713] [3.14] gh-136682: Remove incorrect statement that `os.path.samestat` accepts file-like objects (GH-136683) (#136684) gh-136682: Remove incorrect statement that `os.path.samestat` accepts file-like objects (GH-136683) (cherry picked from commit 7e10a103dfe52feb0ef3d541e08abc2640838101) Co-authored-by: Ran Benita --- Doc/library/os.path.rst | 3 --- 1 file changed, 3 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 32a2970d2d3a2c..3f2574e55699bf 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -508,9 +508,6 @@ the :mod:`glob` module.) .. versionchanged:: 3.4 Added Windows support. - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - .. function:: split(path) From 9ae74e94a06865aee96827838ce16c8924cbe86f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 17:10:37 +0200 Subject: [PATCH 642/713] [3.14] gh-136396: Include instrumentation when creating new copies of the bytecode (GH-136525) (GH-136657) Previously, we assumed that instrumentation would happen for all copies of the bytecode if the instrumentation version on the code object didn't match the per-interpreter instrumentation version. That assumption was incorrect: instrumentation will exit early if there are no new "events," even if there is an instrumentation version mismatch. To fix this, include the instrumented opcodes when creating new copies of the bytecode, rather than replacing them with their uninstrumented variants. I don't think we have to worry about races between instrumentation and creating new copies of the bytecode: instrumentation and new bytecode creation cannot happen concurrently. Instrumentation requires that either the world is stopped or the code object's per-object lock is held and new bytecode creation requires holding the code object's per-object lock. (cherry picked from commit d995922198304a6de19ac1bec3e36d1e886d8468) Co-authored-by: mpage Co-authored-by: Kumar Aditya --- .../test_free_threading/test_monitoring.py | 141 ++++++++++++++++++ ...-07-10-15-53-16.gh-issue-136525.xAko0e.rst | 2 + Objects/codeobject.c | 19 ++- 3 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index a480e398722c33..c3d0a2bcea5c17 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -2,10 +2,12 @@ environment to verify things are thread-safe in a free-threaded build""" import sys +import threading import time import unittest import weakref +from contextlib import contextmanager from sys import monitoring from test.support import threading_helper from threading import Thread, _PyRLock, Barrier @@ -192,6 +194,16 @@ def during_threads(self): self.set = not self.set +class TraceBuf: + def __init__(self): + self.traces = [] + self.traces_lock = threading.Lock() + + def append(self, trace): + with self.traces_lock: + self.traces.append(trace) + + @threading_helper.requires_working_threading() class MonitoringMisc(MonitoringTestMixin, TestCase): def register_callback(self, barrier): @@ -246,6 +258,135 @@ def f(): finally: sys.settrace(None) + def test_toggle_setprofile_no_new_events(self): + # gh-136396: Make sure that profile functions are called for newly + # created threads when profiling is toggled but the set of monitoring + # events doesn't change + traces = [] + + def profiler(frame, event, arg): + traces.append((frame.f_code.co_name, event, arg)) + + def a(x, y): + return b(x, y) + + def b(x, y): + return max(x, y) + + sys.setprofile(profiler) + try: + a(1, 2) + finally: + sys.setprofile(None) + traces.clear() + + def thread_main(x, y): + sys.setprofile(profiler) + try: + a(x, y) + finally: + sys.setprofile(None) + t = Thread(target=thread_main, args=(100, 200)) + t.start() + t.join() + + expected = [ + ("a", "call", None), + ("b", "call", None), + ("b", "c_call", max), + ("b", "c_return", max), + ("b", "return", 200), + ("a", "return", 200), + ("thread_main", "c_call", sys.setprofile), + ] + self.assertEqual(traces, expected) + + def observe_threads(self, observer, buf): + def in_child(ident): + return ident + + def child(ident): + with observer(): + in_child(ident) + + def in_parent(ident): + return ident + + def parent(barrier, ident): + barrier.wait() + with observer(): + t = Thread(target=child, args=(ident,)) + t.start() + t.join() + in_parent(ident) + + num_threads = 5 + barrier = Barrier(num_threads) + threads = [] + for i in range(num_threads): + t = Thread(target=parent, args=(barrier, i)) + t.start() + threads.append(t) + for t in threads: + t.join() + + for i in range(num_threads): + self.assertIn(("in_parent", "return", i), buf.traces) + self.assertIn(("in_child", "return", i), buf.traces) + + def test_profile_threads(self): + buf = TraceBuf() + + def profiler(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + + @contextmanager + def profile(): + sys.setprofile(profiler) + try: + yield + finally: + sys.setprofile(None) + + self.observe_threads(profile, buf) + + def test_trace_threads(self): + buf = TraceBuf() + + def tracer(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + return tracer + + @contextmanager + def trace(): + sys.settrace(tracer) + try: + yield + finally: + sys.settrace(None) + + self.observe_threads(trace, buf) + + def test_monitor_threads(self): + buf = TraceBuf() + + def monitor_py_return(code, off, retval): + buf.append((code.co_name, "return", retval)) + + monitoring.register_callback( + self.tool_id, monitoring.events.PY_RETURN, monitor_py_return + ) + + monitoring.set_events( + self.tool_id, monitoring.events.PY_RETURN + ) + + @contextmanager + def noop(): + yield + + self.observe_threads(noop, buf) + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst new file mode 100644 index 00000000000000..f28eb2ca3b71e8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst @@ -0,0 +1,2 @@ +Fix issue where per-thread bytecode was not instrumented for newly created +threads. diff --git a/Objects/codeobject.c b/Objects/codeobject.c index ba178abc0c071e..42e021679b583f 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -3330,12 +3330,29 @@ _PyCodeArray_New(Py_ssize_t size) return arr; } +// Get the underlying code unit, leaving instrumentation +static _Py_CODEUNIT +deopt_code_unit(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; + int opcode = inst.op.code; + if (opcode < MIN_INSTRUMENTED_OPCODE) { + inst.op.code = _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + } + // JIT should not be enabled with free-threading + assert(inst.op.code != ENTER_EXECUTOR); + return inst; +} + static void copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) { int code_len = (int) Py_SIZE(co); for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { - dst[i] = _Py_GetBaseCodeUnit(co, i); + dst[i] = deopt_code_unit(co, i); } _PyCode_Quicken(dst, code_len, 1); } From 323974295d099d9a9ff87887d5e4116b01367141 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 17:58:52 +0200 Subject: [PATCH 643/713] [3.14] Fix index entry and anchor for module.__test__ (GH-136674) (GH-136688) It was "doctest.module attribute". Now it is "module attribute". (cherry picked from commit 7689407fa4406ab79d7e9e02363f50be4ec35b5e) Co-authored-by: Serhiy Storchaka --- Doc/library/doctest.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 8236d703fc1e45..82e570a2a85cd7 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -311,9 +311,13 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. currentmodule:: None + .. attribute:: module.__test__ :no-typesetting: +.. currentmodule:: doctest + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other From d888f46fba3defa9fa3b39509827768bf3ac01ff Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 15 Jul 2025 19:07:01 +0200 Subject: [PATCH 644/713] [3.14] Fix the doctest.testmod() docstring (GH-136675) (GH-136690) __test__ = None is not supported since Python 2.4. (cherry picked from commit cb59eaefeda5ff44ac0c742bff2b8afc023be313) Co-authored-by: Serhiy Storchaka --- Lib/doctest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/doctest.py b/Lib/doctest.py index dec10a345165da..8860bed2a9f710 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -1991,8 +1991,8 @@ def testmod(m=None, name=None, globs=None, verbose=None, from module m (or the current module if m is not supplied), starting with m.__doc__. - Also test examples reachable from dict m.__test__ if it exists and is - not None. m.__test__ maps names to functions, classes and strings; + Also test examples reachable from dict m.__test__ if it exists. + m.__test__ maps names to functions, classes and strings; function and class docstrings are tested even if the name is private; strings are tested directly, as if they were docstrings. From cdf98c3b7ebca89ae2ece1d30f21ee4adc0ef92e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 06:50:51 +0200 Subject: [PATCH 645/713] [3.14] gh-127146: Report uid in Emscripten + node as native uid (GH-136509) (#136699) Corrects the handling of getuid on emscripten, which was consistently reporting as 0. (cherry picked from commit e81c4e84b3a8688a367099e3adf9b2fcf914447f) Co-authored-by: Hood Chatham --- Python/emscripten_syscalls.c | 19 +++++++++++++++++++ Tools/c-analyzer/cpython/_parser.py | 1 + configure | 2 +- configure.ac | 2 +- 4 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 Python/emscripten_syscalls.c diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c new file mode 100644 index 00000000000000..7875bfc8fe56ae --- /dev/null +++ b/Python/emscripten_syscalls.c @@ -0,0 +1,19 @@ +#include "emscripten.h" + +// If we're running in node, report the UID of the user in the native system as +// the UID of the user. Since the nodefs will report the uid correctly, if we +// don't make getuid report it correctly too we'll see some permission errors. +// Normally __syscall_getuid32 is a stub that always returns 0 but it is +// defined with weak linkage so we can override it. +EM_JS(int, __syscall_getuid32_js, (void), { + // If we're in node and we can, report the native uid + if (typeof process !== "undefined" && typeof process.getuid === "function") { + return process.getuid(); + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_getuid32(void) { + return __syscall_getuid32_js(); +} diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c7..cfbf0d14348499 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -66,6 +66,7 @@ def clean_lines(text): Python/dynload_dl.c # dl.h Python/dynload_hpux.c # dl.h Python/emscripten_signal.c +Python/emscripten_syscalls.c Python/thread_pthread.h Python/thread_pthread_stubs.h diff --git a/configure b/configure index 67a40b841aa506..2242865313e57c 100755 --- a/configure +++ b/configure @@ -19077,7 +19077,7 @@ PLATFORM_OBJS= case $ac_sys_system in #( Emscripten) : - as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o' + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_syscalls.o' as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h' ;; #( *) : diff --git a/configure.ac b/configure.ac index bbd0fa8c1ae8f6..a05b3b18efec36 100644 --- a/configure.ac +++ b/configure.ac @@ -5131,7 +5131,7 @@ PLATFORM_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ - AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o']) + AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_syscalls.o']) AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h']) ], ) From 677b0e314ccbb7f5cae014fa4edd944f6e5bf857 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 09:26:52 +0200 Subject: [PATCH 646/713] [3.14] gh-136523: Fix wave.Wave_write emitting an unraisable when open raises (GH-136529) (GH-136606) (cherry picked from commit 171de05b4884d1353044417ea51a4efcb55ba633) Co-authored-by: Sachin Shah <39803835+inventshah@users.noreply.github.com> --- Lib/test/test_wave.py | 9 +++++++++ Lib/wave.py | 2 ++ .../2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst | 1 + 3 files changed, 12 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 5e771c8de969ec..346a343761a7c1 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,7 @@ from test import audiotests from test import support import io +import os import struct import sys import wave @@ -222,6 +223,14 @@ def test_read_wrong_sample_width(self): with self.assertRaisesRegex(wave.Error, 'bad sample width'): wave.open(io.BytesIO(b)) + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + if __name__ == '__main__': unittest.main() diff --git a/Lib/wave.py b/Lib/wave.py index a34af244c3e224..b8476e264868fc 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -441,6 +441,8 @@ class Wave_write: _datawritten -- the size of the audio samples actually written """ + _file = None + def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): diff --git a/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst b/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst new file mode 100644 index 00000000000000..71ec66a37ef4c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst @@ -0,0 +1 @@ +Fix :class:`wave.Wave_write` emitting an unraisable when open raises. From 9ab29cac192b7678ecb133bd743c295f1d8617f5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:01:44 +0200 Subject: [PATCH 647/713] [3.14] Add `.gram` file to the `.editorconfig` (GH-136680) (#136714) Add `.gram` file to the `.editorconfig` (GH-136680) (cherry picked from commit 2f0db9b05f0598548c0c136571c31065ecf961e5) Co-authored-by: sobolevn --- .editorconfig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.editorconfig b/.editorconfig index 5b04b32a89e3d2..25bc5935258bd1 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h}] +[*.{py,c,cpp,h,gram}] indent_size = 4 [*.rst] From ca6db4fdae9e2a7369502218b46b03c2624a0836 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 17:54:04 +0200 Subject: [PATCH 648/713] [3.14] gh-127146: Emscripten: more regular stack overflow skips (GH-136708) (#136712) Makes the Emscripten stack overflow skip message consistent with WASI, and replaces some ad-hoc skips. (cherry picked from commit c730952aa64b790c75c437cb63a1242dc08c2e97) Co-authored-by: Hood Chatham --- Lib/test/support/__init__.py | 2 +- Lib/test/test_descr.py | 2 +- Lib/test/test_xml_etree_c.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index ce18518bb8aa18..001ecec4dcd4eb 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -570,7 +570,7 @@ def skip_android_selinux(name): is_wasi = sys.platform == "wasi" def skip_emscripten_stack_overflow(): - return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") def skip_wasi_stack_overflow(): return unittest.skipIf(is_wasi, "Exhausts stack on WASI") diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index ea076ba4fef2db..d420f097e74721 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3942,7 +3942,7 @@ def __del__(self): # it as a leak. del C.__del__ - @unittest.skipIf(support.is_emscripten, "Seems to works in Pyodide?") + @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def test_slots_trash(self): # Testing slot trash... diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 9ed0f4096a45e3..270b9d6da8e7b9 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -58,7 +58,7 @@ def test_del_attribute(self): self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) @support.skip_wasi_stack_overflow() - @unittest.skipIf(support.is_emscripten, "segfaults") + @support.skip_emscripten_stack_overflow() def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') From 4eee754091eb1f4cf0d989d5d780ca8da45bb0ed Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:12:36 +0200 Subject: [PATCH 649/713] [3.14] gh-127146: Emscripten: Make os.umask() actually work (GH-136706) (#136711) Provide a stub implementation of umask that is enough to get some tests passing. More work is needed upstream in Emscripten to make all umask tests to pass. (cherry picked from commit 12e52cad718723636a96042f9399634392285c44) Co-authored-by: Hood Chatham --- Lib/test/test_os.py | 6 ++---- Python/emscripten_syscalls.c | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 1e50dc43c35f5c..de3a17fe893170 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1918,11 +1918,9 @@ def test_makedir(self): support.is_wasi, "WASI's umask is a stub." ) - @unittest.skipIf( - support.is_emscripten, - "TODO: Fails in buildbot; see #135783" - ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index 7875bfc8fe56ae..bb80f979420ec1 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -7,7 +7,7 @@ // defined with weak linkage so we can override it. EM_JS(int, __syscall_getuid32_js, (void), { // If we're in node and we can, report the native uid - if (typeof process !== "undefined" && typeof process.getuid === "function") { + if (ENVIRONMENT_IS_NODE) { return process.getuid(); } // Fall back to the stub case of returning 0. @@ -17,3 +17,23 @@ EM_JS(int, __syscall_getuid32_js, (void), { int __syscall_getuid32(void) { return __syscall_getuid32_js(); } + +EM_JS(int, __syscall_umask_js, (int mask), { + // If we're in node and we can, call native process.umask() + if (ENVIRONMENT_IS_NODE) { + try { + return process.umask(mask); + } catch(e) { + // oops... + // NodeJS docs: "In Worker threads, process.umask(mask) will throw an exception." + // umask docs: "This system call always succeeds" + return 0; + } + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_umask(int mask) { + return __syscall_umask_js(mask); +} From 9b7c4183cdfe245c06d417e871452398e4be1935 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:24:15 +0200 Subject: [PATCH 650/713] [3.14] gh-127146: Emscripten: Don't need to avoid unpaired surrogate anymore (GH-136707) (#136717) This might have been fixed by gh-136624, or by some Emscripten change. In any case, it no longer seems to be needed. (cherry picked from commit dcd27aace180737adaddc79c00c181816fc6e162) Co-authored-by: Hood Chatham --- Lib/test/test_warnings/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 5c3b1250ceb045..f89e94449b3031 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -555,13 +555,7 @@ def test_warn_explicit_non_ascii_filename(self): with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - filenames = ["nonascii\xe9\u20ac"] - if not support.is_emscripten: - # JavaScript does not like surrogates. - # Invalid UTF-8 leading byte 0x80 encountered when - # deserializing a UTF-8 string in wasm memory to a JS - # string! - filenames.append("surrogate\udc80") + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] for filename in filenames: try: os.fsencode(filename) From 6943d8ef275a5b4a687a730287d9421e2fae3fae Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Wed, 16 Jul 2025 18:40:11 +0200 Subject: [PATCH 651/713] [3.14] gh-126548: Add a thread-unsafety warning for `importlib.reload()` (GH-136704) (GH-136723) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-126548: Add a thread-unsafety warning for `importlib.reload()` (GH-136704) (cherry picked from commit 69d8fe50ddc4dbe757c9929a532e2e882f0261ba) Co-authored-by: Bartosz Sławecki --- Doc/library/importlib.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index ea5a77028683b3..4f374be778d6b3 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -206,6 +206,10 @@ Functions :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks a :class:`~importlib.machinery.ModuleSpec`. + .. warning:: + This function is not thread-safe. Calling it from multiple threads can result + in unexpected behavior. It's recommended to use the :class:`threading.Lock` + or other synchronization primitives for thread-safe module reloading. :mod:`importlib.abc` -- Abstract base classes related to import --------------------------------------------------------------- From 5f1d46f7d0b03d4deb192e37e6a727136e2e0aa3 Mon Sep 17 00:00:00 2001 From: Zachary Ware Date: Wed, 16 Jul 2025 12:15:50 -0500 Subject: [PATCH 652/713] [3.14] gh-136710: Fix bad indentation in `os.chdir` docstring (GH-136719) (cherry picked from commit bde808ad6ba5eee8a6201983cf071449d7ce7e39) Co-authored-by: Harmen Stoppels --- Modules/clinic/posixmodule.c.h | 4 ++-- Modules/posixmodule.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6b8cc3d07ab01c..65d3aff18d7b42 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -380,7 +380,7 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "path may always be specified as a string.\n" "On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"If this functionality is unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -13398,4 +13398,4 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=f7b5635e0b948be4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c693071966d11548 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 27352f0c20dd48..b3fe9953264ae3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -3477,12 +3477,12 @@ Change the current working directory to the specified path. path may always be specified as a string. On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +If this functionality is unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=1a4a15b4d12cb15d]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=a74ceab5d72adf74]*/ { int result; From 9bf3464852f07c1562ebb4130d41c05d8ef5c3a8 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 17 Jul 2025 00:10:46 +0200 Subject: [PATCH 653/713] [3.14] gh-135148: Correctly handle f/t strings with comments and debug expressions (GH-135198) (#136720) --- Lib/test/test_fstring.py | 12 +++ ...-06-06-02-24-42.gh-issue-135148.r-t2sC.rst | 3 + Parser/lexer/lexer.c | 88 ++++++++++++++----- 3 files changed, 83 insertions(+), 20 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index 58a30c8e6ac447..b41e02c3a16379 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -1651,6 +1651,18 @@ def __repr__(self): self.assertEqual(f"{1+2 = # my comment }", '1+2 = \n 3') + self.assertEqual(f'{""" # booo + """=}', '""" # booo\n """=\' # booo\\n \'') + + self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') + self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + + self.assertEqual(f'{ # some comment goes here + """hello"""=}', ' \n """hello"""=\'hello\'') + self.assertEqual(f'{"""# this is not a comment + a""" # this is a comment + }', '# this is not a comment\n a') + # These next lines contains tabs. Backslash escapes don't # work in f-strings. # patchcheck doesn't like these tabs. So the only way to test diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst new file mode 100644 index 00000000000000..9b1f62433b45ed --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst @@ -0,0 +1,3 @@ +Fixed a bug where f-string debug expressions (using =) would incorrectly +strip out parts of strings containing escaped quotes and # characters. Patch +by Pablo Galindo. diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 0a078dd594148c..81363cf8e810fe 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -121,38 +121,88 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } PyObject *res = NULL; - // Check if there is a # character in the expression + // Look for a # character outside of string literals int hash_detected = 0; + int in_string = 0; + char quote_char = 0; + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { + char ch = tok_mode->last_expr_buffer[i]; + + // Skip escaped characters + if (ch == '\\') { + i++; + continue; + } + + // Handle quotes + if (ch == '"' || ch == '\'') { + // The following if/else block works becase there is an off number + // of quotes in STRING tokens and the lexer only ever reaches this + // function with valid STRING tokens. + // For example: """hello""" + // First quote: in_string = 1 + // Second quote: in_string = 0 + // Third quote: in_string = 1 + if (!in_string) { + in_string = 1; + quote_char = ch; + } + else if (ch == quote_char) { + in_string = 0; + } + continue; + } + + // Check for # outside strings + if (ch == '#' && !in_string) { hash_detected = 1; break; } } - + // If we found a # character in the expression, we need to handle comments if (hash_detected) { - Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); + // Allocate buffer for processed result + char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); if (!result) { return -1; } - Py_ssize_t i = 0; - Py_ssize_t j = 0; + Py_ssize_t i = 0; // Input position + Py_ssize_t j = 0; // Output position + in_string = 0; // Whether we're in a string + quote_char = 0; // Current string quote char - for (i = 0, j = 0; i < input_length; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { - // Skip characters until newline or end of string - while (i < input_length && tok_mode->last_expr_buffer[i] != '\0') { - if (tok_mode->last_expr_buffer[i] == '\n') { - result[j++] = tok_mode->last_expr_buffer[i]; - break; - } + // Process each character + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + char ch = tok_mode->last_expr_buffer[i]; + + // Handle string quotes + if (ch == '"' || ch == '\'') { + // See comment above to understand this part + if (!in_string) { + in_string = 1; + quote_char = ch; + } else if (ch == quote_char) { + in_string = 0; + } + result[j++] = ch; + } + // Skip comments + else if (ch == '#' && !in_string) { + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && + tok_mode->last_expr_buffer[i] != '\n') { i++; } - } else { - result[j++] = tok_mode->last_expr_buffer[i]; + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = '\n'; + } + } + // Copy other chars + else { + result[j++] = ch; } + i++; } result[j] = '\0'; // Null-terminate the result string @@ -164,11 +214,9 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { tok_mode->last_expr_size - tok_mode->last_expr_end, NULL ); - } - - if (!res) { + if (!res) { return -1; } token->metadata = res; From 2ce207108c381a1f765d03dffd6205ecc35db5e1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 17 Jul 2025 12:24:54 +0200 Subject: [PATCH 654/713] [3.14] Improved venv docs to indicate that isolation is the default. (GH-136698) (GH-136705) (cherry picked from commit 8e2f4b448380b4c835442534d566618f06e32573) Co-authored-by: Facundo Batista Co-authored-by: Vinay Sajip --- Doc/library/venv.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index f16e24eac08343..de427fbafe71dc 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -22,10 +22,10 @@ The :mod:`!venv` module supports creating lightweight "virtual environments", each with their own independent set of Python packages installed in their :mod:`site` directories. A virtual environment is created on top of an existing -Python installation, known as the virtual environment's "base" Python, and may -optionally be isolated from the packages in the base environment, -so only those explicitly installed in the virtual environment are available. -See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s +Python installation, known as the virtual environment's "base" Python, and by +default is isolated from the packages in the base environment, +so that only those explicitly installed in the virtual environment are +available. See :ref:`sys-path-init-virtual-environments` and :mod:`site`'s :ref:`virtual environments documentation ` for more information. From 76d4a43bd8d1409dc0f67a3c2b52b3437a1bd049 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 17 Jul 2025 18:05:40 +0200 Subject: [PATCH 655/713] [3.14] gh-127146: Emscripten: Set umask to zero in python.sh (GH-136740) (#136745) Clears the umask used during a test of pydoc.apropos when testing on Emscripten. This is to work around a known issue in Emscripten; but it's not clear if the chmod call that is causing the problem is actually testing anything of significance. (cherry picked from commit 22af5d35a620ee44393853036a8450ceb047688e) Co-authored-by: Hood Chatham --- Lib/test/test_pydoc/test_pydoc.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index 281b24eaa36b80..f5ba5e1eb754be 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -1303,6 +1303,11 @@ def test_apropos_with_unreadable_dir(self): @os_helper.skip_unless_working_chmod def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') + if support.is_emscripten: + # Emscripten's readdir implementation is buggy on directories + # with read permission but no execute permission. + old_umask = os.umask(0) + self.addCleanup(os.umask, old_umask) os.mkdir(pkgdir) self.addCleanup(rmtree, pkgdir) init_path = os.path.join(pkgdir, '__init__.py') From 2ea3b75200d1f78d91a1798630bf28c4d7c9dc9e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Thu, 17 Jul 2025 21:00:09 +0200 Subject: [PATCH 656/713] [3.14] gh-136697: Use the standard audit event format for sys.monitoring docs (GH-136747) (#136749) gh-136697: Use the standard audit event format for sys.monitoring docs (GH-136747) (cherry picked from commit 28937d3a21cf8168c853ae43374a8287c21f71c9) Co-authored-by: Tian Gao --- Doc/library/sys.monitoring.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index f62a4011e4144b..0f986aa580b3c9 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -333,6 +333,8 @@ Registering callback functions it is unregistered and returned. Otherwise :func:`register_callback` returns ``None``. + .. audit-event:: sys.monitoring.register_callback func sys.monitoring.register_callback + Functions can be unregistered by calling ``sys.monitoring.register_callback(tool_id, event, None)``. @@ -343,8 +345,6 @@ globally and locally. As such, if an event could be turned on for both global and local events by your code then the callback needs to be written to handle either trigger. -Registering or unregistering a callback function will generate a :func:`sys.audit` event. - Callback function arguments ''''''''''''''''''''''''''' From 201c765d764a56cfaaa5491c6c234348fe72b422 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 11:13:51 +0200 Subject: [PATCH 657/713] [3.14] Fix typo: "occured" =>"occurred" (GH-134928) (#136771) Co-authored-by: Roman <121314722+GameRoMan@users.noreply.github.com> Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Modules/_ctypes/stgdict.c | 2 +- Modules/_interpretersmodule.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f208d2956e429e..ab955a0b824a2f 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,7 +484,7 @@ error:; /* Replace array elements at stginfo->ffi_type_pointer.elements. - Return -1 if error occured. + Return -1 if error occurred. */ int _replace_array_elements(ctypes_state *st, PyObject *layout_fields, diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index e7feaa7f186aee..9426ce72733c28 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -666,7 +666,7 @@ _run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, // Prep and switch interpreters. if (_PyXI_Enter(session, interp, shareables, &result) < 0) { - // If an error occured at this step, it means that interp + // If an error occurred at this step, it means that interp // was not prepared and switched. _PyXI_FreeSession(session); _PyXI_FreeFailure(failure); From 81aab6d51ccf49a7ec420eaead3f6fa900c57b53 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 11:18:55 +0200 Subject: [PATCH 658/713] [3.14] parser_generator.py typo - keywods -> keywords (GH-135014) (#136772) Co-authored-by: chemelnucfin <3982092+chemelnucfin@users.noreply.github.com> --- Tools/peg_generator/pegen/parser_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 52ae743c26b6b8..7dd56f98a652cc 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -56,7 +56,7 @@ def visit_NamedItem(self, item: NamedItem) -> None: class KeywordCollectorVisitor(GrammarVisitor): - """Visitor that collects all the keywods and soft keywords in the Grammar""" + """Visitor that collects all the keywords and soft keywords in the Grammar""" def __init__(self, gen: "ParserGenerator", keywords: Dict[str, int], soft_keywords: Set[str]): self.generator = gen From aeeacdb8b56232239e70b6aed5838b640116dcef Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 11:36:35 +0200 Subject: [PATCH 659/713] [3.14] Docs: Improve example for ``itertools.batched()`` (GH-136775) (#136778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docs: Improve example for ``itertools.batched()`` (GH-136775) The current example `batched('ABCDEFG', n=3) → ABC DEF G` can confuse readers because both, the size of the tuples and the number of tuples are 3. By using a batch size of n=2, it is clearer that the `n` argument refers to the size of the resulting tuples. I.e. the new example is: `batched('ABCDEFG', n=2) → AB CD EF G` (cherry picked from commit 3eecc72ac70943f7e33297eea17803af15322c88) Co-authored-by: RafaelWO <38643099+RafaelWO@users.noreply.github.com> --- Doc/library/itertools.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 00925ae920aad9..aa46920d3526f0 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -47,7 +47,7 @@ Iterator Arguments Results Iterator Arguments Results Example ============================ ============================ ================================================= ============================================================= :func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) → 1 3 6 10 15`` -:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=3) → ABC DEF G`` +:func:`batched` p, n (p0, p1, ..., p_n-1), ... ``batched('ABCDEFG', n=2) → AB CD EF G`` :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` @@ -181,7 +181,7 @@ loops that truncate the stream. Roughly equivalent to:: def batched(iterable, n, *, strict=False): - # batched('ABCDEFG', 3) → ABC DEF G + # batched('ABCDEFG', 2) → AB CD EF G if n < 1: raise ValueError('n must be at least one') iterator = iter(iterable) From ba6ea7c53d9b8de78380a1f0b029af92633b5332 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 11:56:55 +0200 Subject: [PATCH 660/713] [3.14] gh-74598: document that `fnmatch.filterfalse` is affected by cache limitation (GH-136781) (#136782) gh-74598: document that `fnmatch.filterfalse` is affected by cache limitation (GH-136781) (cherry picked from commit 263e451c4114ac98add1f1e8aa9ee030e054bdfd) Co-authored-by: Gergely Elias --- Doc/library/fnmatch.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 12e61bc36f5db0..ee654b7a83e203 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -53,7 +53,7 @@ a :class:`!str` filename, and vice-versa. Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following -functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. .. function:: fnmatch(name, pat) From a91e2bc440b288c4d3ccc51f068f8d591e0eda1b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 12:23:52 +0200 Subject: [PATCH 661/713] [3.14] Fix typo in `Lib/test/test_ast/test_ast.py` (GH-136767) (#136783) Fix typo in `Lib/test/test_ast/test_ast.py` (GH-136767) `ASTOptimiziationTests` -> `ASTOptimizationTests` (cherry picked from commit 60146f4f6f24f37e3bfcb9f101565f6e86cf0146) Co-authored-by: Hunter Hogan --- Lib/test/test_ast/test_ast.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 59263012bc1440..56e170e128b48c 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -3547,7 +3547,7 @@ def test_show_empty_flag(self): self.check_output(source, expect, '--show-empty') -class ASTOptimiziationTests(unittest.TestCase): +class ASTOptimizationTests(unittest.TestCase): def wrap_expr(self, expr): return ast.Module(body=[ast.Expr(value=expr)]) From 8b4275c99aebef2ac34135584aeb40fad5479b2e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 12:26:22 +0200 Subject: [PATCH 662/713] [3.14] gh-136769: Include fixed-width integers in the fundamental data types table (GH-136784) (#136785) gh-136769: Include fixed-width integers in the fundamental data types table (GH-136784) Fixed-sized types, like ``c_int32``, are currently missing from the fundamental data types table in the ``ctypes`` documentation. This commit adds them, and notes that ``c_[u]int8`` is an alias of ``c_[u]byte``. (cherry picked from commit acefb978dcb5dd554e3c49a3015ee5c2ad6bfda1) Co-authored-by: Sina Zel taat <111974143+SZeltaat@users.noreply.github.com> --- Doc/library/ctypes.rst | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 846cece3761858..09f596101b4d1e 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -232,8 +232,24 @@ Fundamental data types +----------------------+------------------------------------------+----------------------------+ | :class:`c_int` | :c:expr:`int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_int8` | :c:type:`int8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int16` | :c:type:`int16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int32` | :c:type:`int32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_int64` | :c:type:`int64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_uint` | :c:expr:`unsigned int` | int | +----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint8` | :c:type:`uint8_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint16` | :c:type:`uint16_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint32` | :c:type:`uint32_t` | int | ++----------------------+------------------------------------------+----------------------------+ +| :class:`c_uint64` | :c:type:`uint64_t` | int | ++----------------------+------------------------------------------+----------------------------+ | :class:`c_long` | :c:expr:`long` | int | +----------------------+------------------------------------------+----------------------------+ | :class:`c_ulong` | :c:expr:`unsigned long` | int | @@ -2524,7 +2540,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2599,7 +2615,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. From 03457ca2accabcf70ed0f0118c2e92823635c465 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 14:57:52 +0200 Subject: [PATCH 663/713] [3.14] gh-135730: Clarify multiprocessing.Queue close() documentation (GH-136803) (GH-136806) gh-135730: Clarify multiprocessing.Queue close() documentation (GH-136803) Add a copy of the text from SimpleQueue.close() --------- (cherry picked from commit f575588ccf27d8d54a1e99cfda944f2614b3255c) Co-authored-by: aggshruti99 Co-authored-by: Petr Viktorin --- Doc/library/multiprocessing.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index b41953d21442c9..a692df41a8154b 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -936,8 +936,13 @@ For an example of the usage of queues for interprocess communication see .. method:: close() - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`~Queue.get`, :meth:`~Queue.put` and :meth:`~Queue.empty` + methods must no longer be called. + + The background thread will quit once it has flushed all buffered data to the pipe. This is called automatically when the queue is garbage collected. From 2a9aeb0f579ff75cd49dd36fb8f62d7a8adb6bfa Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 15:56:40 +0200 Subject: [PATCH 664/713] [3.14] gh-136801: Fix PyREPL syntax highlightning on match cases after multi-line case (GH-136804) (GH-136813) (cherry picked from commit 3a648445337098abf22c7faa296389dab597797c) Co-authored-by: Olga Matoula --- Lib/_pyrepl/utils.py | 4 ++-- Lib/test/test_pyrepl/test_reader.py | 12 +++++++----- .../2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst | 1 + 3 files changed, 10 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index e04fbdc6c8a5c4..fd788c8429e15b 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -241,14 +241,14 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return s in keyword_first_sets_match return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) | TI(T.OP, string="(" | "*" | "-" | "[" | "{") ): return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NAME, string=s) ): diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 1f655264f1c00a..9a02dff7387563 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -375,7 +375,8 @@ def funct(case: str = sys.platform) -> None: ) match case: case "emscripten": print("on the web") - case "ios" | "android": print("on the phone") + case "ios" | "android": + print("on the phone") case _: print('arms around', match.group(1)) """ ) @@ -393,7 +394,8 @@ def funct(case: str = sys.platform) -> None: {o}){z} {K}match{z} case{o}:{z} {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} - {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} """ ) @@ -402,14 +404,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 2**7 + 2**8) - self.assertEqual(reader.cxy, (0, 14)) + self.assertEqual(reader.pos, 396) + self.assertEqual(reader.cxy, (0, 15)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 13, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 14, code_to_events("async "), ) reader, _ = handle_all_events(more_events) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst new file mode 100644 index 00000000000000..5c0813b1a0abda --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst @@ -0,0 +1 @@ +Fix PyREPL syntax highlightning on match cases after multi-line case. Contributed by Olga Matoula. From 8d063f3d922850ee2199d0e63a93904e53f02ef1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 16:28:52 +0200 Subject: [PATCH 665/713] [3.14] gh-54732: Make argparse error caused by empty rows in option files explicit (GH-136795) (#136818) gh-54732: Make argparse error caused by empty rows in option files explicit (GH-136795) (cherry picked from commit 8ffc3ef01e83ffe629c6107082677de4d23974d5) Co-authored-by: jdunter <2ve@mailbox.org> Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/argparse.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f189f6b8fa8953..a08f713ab56ba3 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -434,12 +434,18 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Empty lines are treated as empty strings (``''``), which are allowed as values but + not as arguments. Empty lines that are read as arguments will result in an + "unrecognized arguments" error. + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. From f8af7cb7231bdb4ae7137c690da4e3903d0ce5f9 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 16:50:30 +0200 Subject: [PATCH 666/713] [3.14] gh-136752: Clarify documentation for ``IPv{N}Address.is_reserved`` (GH-136794) (#136827) gh-136752: Clarify documentation for ``IPv{N}Address.is_reserved`` (GH-136794) (cherry picked from commit 6293d8a1a648a498b7ac899631b74fa25c71c1ac) Co-authored-by: Matthieu Lienart <50069805+mlnrt@users.noreply.github.com> Co-authored-by: Matthieu Lienart Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/ipaddress.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index e5bdfbb144b65a..9e887d8e65741b 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -240,7 +240,16 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_reserved - ``True`` if the address is otherwise IETF reserved. + ``True`` if the address is noted as reserved by the IETF. + For IPv4, this is only ``240.0.0.0/4``, the ``Reserved`` address block. + For IPv6, this is all addresses `allocated `__ as + ``Reserved by IETF`` for future use. + + .. note:: For IPv4, ``is_reserved`` is not related to the address block value of the + ``Reserved-by-Protocol`` column in iana-ipv4-special-registry_. + + .. caution:: For IPv6, ``fec0::/10`` a former Site-Local scoped address prefix is + currently excluded from that list (see :attr:`~IPv6Address.is_site_local` & :rfc:`3879`). .. attribute:: is_loopback @@ -261,6 +270,7 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. _iana-ipv6-address-space: https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml .. method:: IPv4Address.__format__(fmt) From 4606b4089d727ea879e6233ec4a366fffcb28fa1 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 19:36:31 +0200 Subject: [PATCH 667/713] [3.14] gh-136839: Refactor simple dict.update calls (GH-136811) (#136840) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-136839: Refactor simple dict.update calls (GH-136811) Refactor simple dict.update calls This commit refactors simple `dict.update({key: value})` calls which can be done via `dict[key] = value` instead. I found those cases with the [semgrep](https://semgrep.dev/) tool: ``` $ semgrep --lang python --pattern '$DICT.update({$A: ...})' ┌─────────────────┐ │ 5 Code Findings │ └─────────────────┘ Lib/dataclasses.py 1268┆ slots.update({slot: doc}) Lib/multiprocessing/resource_tracker.py 50┆ _CLEANUP_FUNCS.update({ 51┆ 'semaphore': _multiprocessing.sem_unlink, 52┆ }) ⋮┆---------------------------------------- 53┆ _CLEANUP_FUNCS.update({ 54┆ 'shared_memory': _posixshmem.shm_unlink, 55┆ }) Lib/tkinter/scrolledtext.py 26┆ kw.update({'yscrollcommand': self.vbar.set}) Lib/xmlrpc/server.py 242┆ self.funcs.update({'system.multicall' : self.system_multicall}) ``` (cherry picked from commit 69ea1b3a8f45fec46add3272ad47f14ff5321ae8) Co-authored-by: Disconnect3d --- Lib/dataclasses.py | 2 +- Lib/multiprocessing/resource_tracker.py | 8 ++------ Lib/tkinter/scrolledtext.py | 2 +- Lib/xmlrpc/server.py | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 86d29df0639184..83ea623dce6281 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1265,7 +1265,7 @@ def _create_slots(defined_fields, inherited_slots, field_names, weakref_slot): doc = getattr(defined_fields.get(slot), 'doc', None) if doc is not None: seen_docs = True - slots.update({slot: doc}) + slots[slot] = doc # We only return dict if there's at least one doc member, # otherwise we return tuple, which is the old default format. diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 05633ac21a259c..c4d0ca81e7034a 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -47,12 +47,8 @@ def cleanup_noop(name): # absence of POSIX named semaphores. In that case, no named semaphores were # ever opened, so no cleanup would be necessary. if hasattr(_multiprocessing, 'sem_unlink'): - _CLEANUP_FUNCS.update({ - 'semaphore': _multiprocessing.sem_unlink, - }) - _CLEANUP_FUNCS.update({ - 'shared_memory': _posixshmem.shm_unlink, - }) + _CLEANUP_FUNCS['semaphore'] = _multiprocessing.sem_unlink + _CLEANUP_FUNCS['shared_memory'] = _posixshmem.shm_unlink class ReentrantCallError(RuntimeError): diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py index 4f9a8815b6184b..8dcead5e31930e 100644 --- a/Lib/tkinter/scrolledtext.py +++ b/Lib/tkinter/scrolledtext.py @@ -23,7 +23,7 @@ def __init__(self, master=None, **kw): self.vbar = Scrollbar(self.frame) self.vbar.pack(side=RIGHT, fill=Y) - kw.update({'yscrollcommand': self.vbar.set}) + kw['yscrollcommand'] = self.vbar.set Text.__init__(self, self.frame, **kw) self.pack(side=LEFT, fill=BOTH, expand=True) self.vbar['command'] = self.yview diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 8130c739af2fe8..3e6871157d0929 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. From 197f3eca2a48dfa28533645029a926f241e0f25d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sat, 19 Jul 2025 20:00:15 +0200 Subject: [PATCH 668/713] [3.14] gh-136764: improve comment in enum.verify.__call__ (GH-136774) (GH-136841) gh-136764: improve comment in enum.verify.__call__ (GH-136774) (cherry picked from commit 6a1c93af806d0ca5d3fb86cd183d00013bbf28d1) Co-authored-by: Saurav Singh --- Lib/enum.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/enum.py b/Lib/enum.py index 01fecca3e5aac0..8a72c409b94a8c 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -1965,7 +1965,7 @@ def __call__(self, enumeration): if 2**i not in values: missing.append(2**i) elif enum_type == 'enum': - # check for powers of one + # check for missing consecutive integers for i in range(low+1, high): if i not in values: missing.append(i) From cce38eefd4b8f4e90eaa40ebfda64cb5ffde679f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 10:55:06 +0200 Subject: [PATCH 669/713] [3.14] Doc/c-api/memory.rst: extend --without-pymalloc doc with ASan information (GH-136790) (GH-136798) Extend the documentation for disabling pymalloc with the `--without-pymalloc` flag regarding why it is worth to use it when enabling AddressSanitizer for Python build (which is done, e.g., in CPython's CI builds). I have tested the CPython latest main build with both ASan and pymalloc enabled and it seems to work just fine. I did run the `python -m test` suite which didn't uncover any ASan crashes (though, it detected some memory leaks, which I believe are irrelevant here). I have discussed ASan and this flag with @encukou on the CPython Core sprint on EuroPython 2025. We initially thought that the `--without-pymalloc` flag is needed for ASan builds due to the fact pymalloc must hit the begining of page when determining if the memory to be freed comes from pymalloc or was allocated by the system malloc. In other words, we thought, that ASan would crash CPython during free of big objects (allocated by system malloc). It may be that this was the case in the past, but it is not the case anymore as the `address_in_range` function used by pymalloc is annotated to be skipped from the ASan instrumentation. (cherry picked from commit d19bb4471331ca2cb87b86e4c904bc9a2bafb044) Co-authored-by: Disconnect3d Co-authored-by: Petr Viktorin --- Doc/c-api/memory.rst | 4 ++++ Doc/using/configure.rst | 3 +++ 2 files changed, 7 insertions(+) diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 61fa49f8681cce..df1bb0ce370919 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -672,6 +672,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index e5fe3c72b1b26e..2cda9587975ddc 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -802,6 +802,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). + To improve ASan detection capabilities you may also want to combine this + with :option:`--without-pymalloc` to disable the specialized small-object + allocator whose allocations are not tracked by ASan. .. versionadded:: 3.6 From 2cb3b534e724ba49a742586590d3ee65efdc2d71 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 13:28:17 +0200 Subject: [PATCH 670/713] [3.14] gh-108362: Retarget incremental GC changes to 3.14 (GH-125453) (#136851) Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/library/gc.rst | 8 ++--- Doc/whatsnew/3.14.rst | 71 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 7ccb0e6bdf9406..2ef5c4b35a25cc 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -60,7 +60,7 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. @@ -83,13 +83,13 @@ The :mod:`gc` module provides the following functions: returned. If *generation* is not ``None``, return only the objects as follows: * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.13) + * 1: No objects, as there is no generation 1 (as of Python 3.14) * 2: All objects in the old generation .. versionchanged:: 3.8 New *generation* parameter. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Generation 1 is removed .. audit-event:: gc.get_objects generation gc.get_objects @@ -142,7 +142,7 @@ The :mod:`gc` module provides the following functions: See `Garbage collector design `_ for more information. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 *threshold2* is ignored diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index a6fb3e953eae58..ca9aa0b55605f1 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1058,6 +1058,30 @@ free-threaded build and false for the GIL-enabled build. (Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) +.. _whatsnew314-incremental-gc: + +Incremental garbage collection +------------------------------ + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + +There are now only two generations: young and old. +When :func:`gc.collect` is not called directly, the +GC is invoked a little less frequently. When invoked, it +collects the young generation and an increment of the +old generation, instead of collecting one or more generations. + +The behavior of :func:`!gc.collect` changes slightly: + +* ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. +* Other calls to :func:`!gc.collect` are unchanged. + +(Contributed by Mark Shannon in :gh:`108362`.) + + Other language changes ====================== @@ -1486,6 +1510,36 @@ functools (Contributed by Sayandip Dutta in :gh:`125916`.) +gc +-- + +The cyclic garbage collector is now incremental, +which changes the meaning of the results of +:meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` +as well as :meth:`~gc.get_count` and :meth:`~gc.get_stats`. + +* For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is meaningless and is always zero. + +* :meth:`~gc.set_threshold` ignores any items after the second. + +* :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. + +In summary, code that attempted to manipulate the behavior of the cycle GC +may not work exactly as intended, but it is very unlikely to be harmful. +All other code will work just fine. + + getopt ------ @@ -2233,6 +2287,7 @@ asyncio (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa in :gh:`91048`.) + base64 ------ @@ -2241,6 +2296,15 @@ base64 (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) +gc +-- + +* The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. + (Contributed by Mark Shannon in :gh:`108362`.) + + io --- * :mod:`io` which provides the built-in :func:`open` makes less system calls @@ -2707,6 +2771,13 @@ Changes in the Python API Wrap it in :func:`staticmethod` if you want to preserve the old behavior. (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + * The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` locale in some cases. This temporary change affects other threads. From 0e909267f65bb0050a17e4651877ecec747e528a Mon Sep 17 00:00:00 2001 From: Olga Pustovalova <162949+olp-cs@users.noreply.github.com> Date: Sun, 20 Jul 2025 13:51:59 +0200 Subject: [PATCH 671/713] [3.14] gh-136438: Make sure `test_remote_pdb` pass with all optimization levels (GH-136788) (GH-136855) (cherry picked from commit 588d9fb84ae014502811ec8580411ea0df7200fe) --- Lib/test/test_remote_pdb.py | 62 +++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index aef8a6b0129092..280e2444ef7d34 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1,5 +1,4 @@ import io -import time import itertools import json import os @@ -8,16 +7,13 @@ import socket import subprocess import sys -import tempfile import textwrap -import threading import unittest import unittest.mock from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack -from pathlib import Path -from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT -from test.support.os_helper import temp_dir, TESTFN, unlink -from typing import Dict, List, Optional, Tuple, Union, Any +from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT, subTests +from test.support.os_helper import TESTFN, unlink +from typing import List import pdb from pdb import _PdbServer, _PdbClient @@ -283,37 +279,50 @@ def test_handling_other_message(self): expected_stdout="Some message.\n", ) - def test_handling_help_for_command(self): - """Test handling a request to display help for a command.""" + @unittest.skipIf(sys.flags.optimize >= 2, "Help not available for -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "Usage: ll | longlist"), + # a request to display a help overview + ({"help": ""}, "type help "), + # a request to display the full PDB manual + ({"help": "pdb"}, ">>> import pdb"), + ], + ) + def test_handling_help_when_available(self, help_request, expected_substring): + """Test handling help requests when help is available.""" incoming = [ - ("server", {"help": "ll"}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="Usage: ll | longlist", + expected_stdout_substring=expected_substring, ) - def test_handling_help_without_a_specific_topic(self): - """Test handling a request to display a help overview.""" + @unittest.skipIf(sys.flags.optimize < 2, "Needs -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "No help for 'll'"), + # a request to display a help overview + ({"help": ""}, "Undocumented commands"), + # a request to display the full PDB manual + ({"help": "pdb"}, "No help for 'pdb'"), + ], + ) + def test_handling_help_when_not_available(self, help_request, expected_substring): + """Test handling help requests when help is not available.""" incoming = [ - ("server", {"help": ""}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="type help ", - ) - - def test_handling_help_pdb(self): - """Test handling a request to display the full PDB manual.""" - incoming = [ - ("server", {"help": "pdb"}), - ] - self.do_test( - incoming=incoming, - expected_outgoing=[], - expected_stdout_substring=">>> import pdb", + expected_stdout_substring=expected_substring, ) def test_handling_pdb_prompts(self): @@ -1434,7 +1443,6 @@ def test_multi_line_commands(self): def _supports_remote_attaching(): - from contextlib import suppress PROCESS_VM_READV_SUPPORTED = False try: From ee37365e0db5712b808456d136c19ef582059bbd Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 14:13:48 +0200 Subject: [PATCH 672/713] [3.14] gh-136854: Exit on error in `make venv` (GH-136856) (#136860) Co-authored-by: Nacho Caballero Co-authored-by: Nacho Caballero --- Doc/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Doc/Makefile b/Doc/Makefile index c8a749a02a89ec..84578c5c57f478 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -170,6 +170,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ $(UV) venv --python=$(PYTHON) $(VENVDIR); \ From c537b98e489501e6ad33f244b9eb1e78db1e1bf2 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 14:32:58 +0200 Subject: [PATCH 673/713] [3.14] gh-130655: gettext: Add fallback testcase (GH-136857) (#136862) gh-130655: gettext: Add fallback testcase (GH-136857) (cherry picked from commit c6e6fe92cd8b90d546652764e3eaf1631da16f8f) Co-authored-by: Dominic H --- Lib/test/test_gettext.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 33b7d75e3ff203..9ad37909a8ec4e 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -937,6 +937,13 @@ def test_lazy_import(self): ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + + if __name__ == '__main__': unittest.main() From 2caae152b962ea67ebc8bb473d7d594152f0e443 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 15:16:19 +0200 Subject: [PATCH 674/713] [3.14] gh-86608: Improve and restructure tarfile examples (GH-121771) (#136866) gh-86608: Improve and restructure tarfile examples (GH-121771) Add an example on how to write a tarfile to stdout; general improvements. (cherry picked from commit cc81b4e501138b5793d419c81c3a2859a17207a7) Co-authored-by: Dominic H Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/library/tarfile.rst | 40 ++++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 8d10db8f2c2921..add9e8ac2cb587 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -1353,6 +1353,9 @@ Command-line options Examples -------- +Reading examples +~~~~~~~~~~~~~~~~~~~ + How to extract an entire tar archive to the current working directory:: import tarfile @@ -1375,6 +1378,23 @@ a generator function instead of a list:: tar.extractall(members=py_files(tar)) tar.close() +How to read a gzip compressed tar archive and display some member information:: + + import tarfile + tar = tarfile.open("sample.tar.gz", "r:gz") + for tarinfo in tar: + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") + if tarinfo.isreg(): + print("a regular file.") + elif tarinfo.isdir(): + print("a directory.") + else: + print("something else.") + tar.close() + +Writing examples +~~~~~~~~~~~~~~~~ + How to create an uncompressed tar archive from a list of filenames:: import tarfile @@ -1390,19 +1410,15 @@ The same example using the :keyword:`with` statement:: for name in ["foo", "bar", "quux"]: tar.add(name) -How to read a gzip compressed tar archive and display some member information:: +How to create and write an archive to stdout using +:data:`sys.stdout.buffer ` in the *fileobj* parameter +in :meth:`TarFile.add`:: - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() + import sys + import tarfile + with tarfile.open("sample.tar.gz", "w|gz", fileobj=sys.stdout.buffer) as tar: + for name in ["foo", "bar", "quux"]: + tar.add(name) How to create an archive and reset the user information using the *filter* parameter in :meth:`TarFile.add`:: From 2f7684ceeef487bedbbae3283ad92bbb9d7e143e Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Sun, 20 Jul 2025 22:28:28 +0200 Subject: [PATCH 675/713] [3.14] GH-111758: Merge TSan and UBSan reusable GHA workflows (GH-136820) (#136883) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) Co-authored-by: Sviatoslav Sydorenko Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- .github/workflows/build.yml | 22 +++-- .github/workflows/reusable-san.yml | 124 ++++++++++++++++++++++++++++ .github/workflows/reusable-tsan.yml | 94 --------------------- 3 files changed, 139 insertions(+), 101 deletions(-) create mode 100644 .github/workflows/reusable-san.yml delete mode 100644 .github/workflows/reusable-tsan.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e26433786f74e4..bb29a6760e17cd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -557,20 +557,28 @@ jobs: - name: Tests run: xvfb-run make ci - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + build-san: + name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category + Sanitizers${{ '' }} needs: build-context if: needs.build-context.outputs.run-tests == 'true' strategy: fail-fast: false matrix: + check-name: + - Thread free-threading: - false - true - uses: ./.github/workflows/reusable-tsan.yml + sanitizer: + - TSan + include: + - check-name: Undefined behavior + sanitizer: UBSan + free-threading: false + uses: ./.github/workflows/reusable-san.yml with: + sanitizer: ${{ matrix.sanitizer }} config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} @@ -671,7 +679,7 @@ jobs: - build-wasi - test-hypothesis - build-asan - - build-tsan + - build-san - cross-build-linux - cifuzz if: always() @@ -704,7 +712,7 @@ jobs: build-wasi, test-hypothesis, build-asan, - build-tsan, + build-san, cross-build-linux, ' || '' diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml new file mode 100644 index 00000000000000..e6ff02e4838ee6 --- /dev/null +++ b/.github/workflows/reusable-san.yml @@ -0,0 +1,124 @@ +name: Reusable Sanitizer + +on: + workflow_call: + inputs: + sanitizer: + required: true + type: string + config_hash: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +env: + FORCE_COLOR: 1 + +jobs: + build-san-reusable: + name: >- + ${{ inputs.sanitizer }}${{ + inputs.free-threading + && ' (free-threading)' + || '' + }} + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Restore config.cache + uses: actions/cache@v4 + with: + path: config.cache + key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.sanitizer }}-${{ inputs.config_hash }} + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + + if [ "${SANITIZER}" = "TSan" ]; then + sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 + sudo update-alternatives --set clang /usr/bin/clang-17 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-17 + # Reduce ASLR to avoid TSan crashing + sudo sysctl -w vm.mmap_rnd_bits=28 + else + sudo ./llvm.sh 20 + fi + + - name: Sanitizer option setup + run: | + if [ "${SANITIZER}" = "TSan" ]; then + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }}.txt handle_segv=0" >> "$GITHUB_ENV" + else + echo "UBSAN_OPTIONS=${SAN_LOG_OPTION}" >> "$GITHUB_ENV" + fi + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + env: + SANITIZER: ${{ inputs.sanitizer }} + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1.2 + with: + save: ${{ github.event_name == 'push' }} + max-size: "200M" + - name: Configure CPython + run: >- + ./configure + --config-cache + ${{ + inputs.sanitizer == 'TSan' + && '--with-thread-sanitizer' + || '--with-undefined-behavior-sanitizer' + }} + --with-pydebug + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: >- + ./python -m test + ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + -j4 + - name: Parallel tests + if: >- + inputs.sanitizer == 'TSan' + && fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 + - name: Display logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 + - name: Archive logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: >- + ${{ inputs.sanitizer }}-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: san_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml deleted file mode 100644 index 6a58e5305f8e09..00000000000000 --- a/.github/workflows/reusable-tsan.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Reusable Thread Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - free-threading: - description: Whether to use free-threaded mode - required: false - type: boolean - default: false - -env: - FORCE_COLOR: 1 - -jobs: - build-tsan-reusable: - name: 'Thread sanitizer' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - - name: Install dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-18 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN option setup - run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: >- - ./configure - --config-cache - --with-thread-sanitizer - --with-pydebug - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 - - name: Parallel tests - if: fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 - - name: Display TSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - - name: Archive TSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: >- - tsan-logs-${{ - fromJSON(inputs.free-threading) - && 'free-threading' - || 'default' - }} - path: tsan_log.* - if-no-files-found: ignore From 17c5959aa3daee4b63d78e944a1d355daa651e52 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 00:22:53 +0200 Subject: [PATCH 676/713] [3.14] GH-130645: Default to color help in argparse (GH-136809) (#136886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GH-130645: Default to color help in argparse (GH-136809) (cherry picked from commit acbe896cb12d6a92e6150fff22921756f6928035) Co-authored-by: Pablo Galindo Salgado Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Łukasz Langa Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/library/argparse.rst | 24 +++++++------------ Doc/whatsnew/3.14.rst | 9 ++++--- Lib/argparse.py | 6 ++--- Lib/test/test_argparse.py | 23 +++++++++++++++++- Lib/test/test_clinic.py | 1 + ...-07-19-16-20-54.gh-issue-130645.O-dYcN.rst | 1 + 6 files changed, 39 insertions(+), 25 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index a08f713ab56ba3..79e15994491eff 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -74,7 +74,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=False) + *, suggest_on_error=False, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -119,7 +119,7 @@ ArgumentParser objects * suggest_on_error_ - Enables suggestions for mistyped argument choices and subparser names (default: ``False``) - * color_ - Allow color output (default: ``False``) + * color_ - Allow color output (default: ``True``) .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -626,27 +626,19 @@ keyword argument:: color ^^^^^ -By default, the help message is printed in plain text. If you want to allow -color in help messages, you can enable it by setting ``color`` to ``True``:: +By default, the help message is printed in color using `ANSI escape sequences +`__. +If you want plain text help messages, you can disable this :ref:`in your local +environment `, or in the argument parser itself +by setting ``color`` to ``False``:: >>> parser = argparse.ArgumentParser(description='Process some integers.', - ... color=True) + ... color=False) >>> parser.add_argument('--action', choices=['sum', 'max']) >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) -Even if a CLI author has enabled color, it can be -:ref:`controlled using environment variables `. - -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``color`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.color = True - .. versionadded:: 3.14 diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index ca9aa0b55605f1..b85440736963cb 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1228,11 +1228,10 @@ argparse .. _whatsnew314-color-argparse: -* Introduced the optional *color* parameter to - :class:`argparse.ArgumentParser`, enabling color for help text. - This can be controlled by :ref:`environment variables - `. Color has also been enabled for help in the - :ref:`stdlib CLIs ` which use :mod:`!argparse`. +* Enable color for help text, which can be disabled with the optional *color* + parameter to :class:`argparse.ArgumentParser`. + This can also be controlled by :ref:`environment variables + `. (Contributed by Hugo van Kemenade in :gh:`130645`.) diff --git a/Lib/argparse.py b/Lib/argparse.py index 83258cf3e0f37d..2144c81886ad19 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -167,7 +167,7 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - color=False, + color=True, ): # default setting for width if width is None: @@ -1231,7 +1231,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() - self._color = False + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1878,7 +1878,7 @@ def __init__(self, exit_on_error=True, *, suggest_on_error=False, - color=False, + color=True, ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 08ff41368d9bb0..2f39b42ab74299 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -18,7 +18,11 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, +) from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -1007,6 +1011,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -2403,6 +2408,7 @@ def test_modified_invalid_action(self): # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -3009,6 +3015,7 @@ def test_nested_argument_group(self): # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -3216,6 +3223,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3344,21 +3352,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -4030,11 +4042,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4047,6 +4061,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4788,6 +4803,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -5469,6 +5485,7 @@ def custom_type(string): version = '' +@force_not_colorized_test_class class TestHelpCustomHelpFormatter(TestCase): maxDiff = None @@ -5765,6 +5782,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -6031,6 +6049,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6829,6 +6848,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6994,6 +7014,7 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class class TestProgName(TestCase): source = textwrap.dedent('''\ import argparse diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 4b1f5991a39ee8..d5b751eafc5bbb 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -2792,6 +2792,7 @@ def test_cli_verbose(self): out = self.expect_success("-v", fn) self.assertEqual(out.strip(), fn) + @support.force_not_colorized def test_cli_help(self): out = self.expect_success("-h") self.assertIn("usage: clinic.py", out) diff --git a/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst b/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst new file mode 100644 index 00000000000000..96e076dfe5bd12 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst @@ -0,0 +1 @@ +Enable color help by default in :mod:`argparse`. From 9663f93a50469828e3c149c6b535227170a29409 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 02:00:19 +0200 Subject: [PATCH 677/713] [3.14] gh-136470: Correct InterpreterPoolExecutor's default thread name (GH-136472) (GH-136889) gh-136470: Correct InterpreterPoolExecutor's default thread name (GH-136472) The OS thread name is now correctly prefixed with `InterpreterPoolExecutor` instead of `ThreadPoolExecutor`. (cherry picked from commit 246be21de1e2a51d757c747902108dfec13e0605) Co-authored-by: AN Long --- Lib/concurrent/futures/interpreter.py | 2 ++ .../test_interpreter_pool.py | 15 +++++++++++++++ ...2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst | 2 ++ 3 files changed, 19 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index cbb60ce80c1813..53c6e757ded2e3 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -118,5 +118,7 @@ def __init__(self, max_workers=None, thread_name_prefix='', each worker interpreter. initargs: A tuple of arguments to pass to the initializer. """ + thread_name_prefix = (thread_name_prefix or + (f"InterpreterPoolExecutor-{self._counter()}")) super().__init__(max_workers, thread_name_prefix, initializer, initargs) diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index d5c032d01cdf5d..7241fcc4b1e74d 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -1,3 +1,4 @@ +import _thread import asyncio import contextlib import io @@ -498,6 +499,20 @@ def test_import_interpreter_pool_executor(self): self.assertEqual(p.stdout.decode(), '') self.assertEqual(p.stderr.decode(), '') + def test_thread_name_prefix(self): + self.assertStartsWith(self.executor._thread_name_prefix, + "InterpreterPoolExecutor-") + + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_thread_name_prefix_with_thread_get_name(self): + def get_thread_name(): + import _thread + return _thread._get_name() + + # Some platforms (Linux) are using 16 bytes to store the thread name, + # so only compare the first 15 bytes (without the trailing \n). + self.assertStartsWith(self.executor.submit(get_thread_name).result(), + "InterpreterPoolExecutor-"[:15]) class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): diff --git a/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst b/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst new file mode 100644 index 00000000000000..5a0429cae07168 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst @@ -0,0 +1,2 @@ +Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread +name. From d0623cbbf5d0c3763112e5477f207c83f29591a0 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 10:29:52 +0200 Subject: [PATCH 678/713] [3.14] gh-136428: amend UUIDv8 performance improvements (GH-136903) (#136904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gh-136428: amend UUIDv8 performance improvements (GH-136903) UUIDv8 has been added in Python 3.14.0a2 and its construction time has been improved in Python 3.14.0a4, but since those changes will not be visible when comparing the latest Python 3.13 and 3.14 together, we do not document them on the What's New page to avoid confusion. (cherry picked from commit 5798348a0739ccf46f690f5fa1443080ec5de310) Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Doc/whatsnew/3.14.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index b85440736963cb..d58f7ecf02ce6b 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2323,8 +2323,7 @@ uuid * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster for 16-byte names and 20% faster for 1024-byte names. Performance for longer names remains unchanged. - * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster - respectively. + * :func:`~uuid.uuid4` is 30% faster. (Contributed by Bénédikt Tran in :gh:`128150`.) From 163f532530efd2c910fa4ee3897ed06b47eeb3b5 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:04:39 +0200 Subject: [PATCH 679/713] [3.14] gh-136882: Update stale link in the basic logging tutorial. (GH-136885) (#136905) Co-authored-by: Vinay Sajip Co-authored-by: Stan Ulbrych <89152624+StanFromIreland@users.noreply.github.com> --- Doc/howto/logging.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 2982cf88bf97b4..b7225ff1c2cbfc 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -302,10 +302,10 @@ reading the following sections. If you're ready for that, grab some of your favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. +logging into your own scripts, and if you run into problems or don't understand +something, please post a question in the Help category of the `Python +discussion forum `_ and you should receive +help before too long. Still here? You can carry on reading the next few sections, which provide a slightly more advanced/in-depth tutorial than the basic one above. After that, From a69472e242e768545d734cf336bdbbff6f749f3f Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:23:33 +0200 Subject: [PATCH 680/713] [3.14] gh-134411: assert `PyLong_FromLong(x) != NULL` when `x` is known to be small (GH-134415) (#136910) gh-134411: assert `PyLong_FromLong(x) != NULL` when `x` is known to be small (GH-134415) Since `PyLong_From Long(PY_MONITORING_DEBUGGER_ID)` falls to `small_int` case and can't return `NULL`. Added `assert`s for extra confidence. https://github.com/python/cpython/issues/134411#issuecomment-2897653868 (cherry picked from commit cf19b6435d02dd7be11b84a44f4a8a9f1a935b15) Co-authored-by: Sergey Muraviov --- Python/instrumentation.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 13bdd041becd69..5b26a078085806 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -2558,18 +2558,22 @@ PyObject *_Py_CreateMonitoringObject(void) err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + assert(val != NULL); /* Can't return NULL because the int is small. */ err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "PROFILER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); Py_DECREF(val); if (err) goto error; From 4f6f3ee8d3710fef102071c53919a9af7a28613b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 11:58:57 +0200 Subject: [PATCH 681/713] [3.14] gh-136852: Emscripten: Add PYTHON_NODE_VERSION environment variable (GH-136853) (GH-136907) To choose the node version we use. Together with: https://github.com/python/buildmaster-config/pull/614 closes GH-136852. (cherry picked from commit aec7f5f8b2e8b5e02869cdb4e1f8a9ef87c9f953) Co-authored-by: Hood Chatham --- Tools/wasm/emscripten/__main__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index c0d58aeaadd2cf..1b31e36dd890b3 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -206,6 +206,17 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner + if node_version := os.environ.get("PYTHON_NODE_VERSION", None): + res = subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm which {node_version}", + ], + text=True, + capture_output=True, + ) + host_runner = res.stdout.strip() pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() env_additions = { "CONFIG_SITE": config_site, From 031645a88427d8c8df70ae96519b4d2d0ae550f7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 13:02:41 +0200 Subject: [PATCH 682/713] [3.14] gh-135621: Remove dependency on curses from PyREPL (GH-136758) (GH-136915) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (cherry picked from commit 09dfb50f1b7c23bc48d86bd579671761bb8ca48b) Co-authored-by: Łukasz Langa --- Lib/_pyrepl/_minimal_curses.py | 68 -- Lib/_pyrepl/curses.py | 33 - Lib/_pyrepl/terminfo.py | 530 ++++++++++++++ Lib/_pyrepl/unix_console.py | 16 +- Lib/_pyrepl/unix_eventqueue.py | 11 +- Lib/test/test_pyrepl/__init__.py | 20 +- Lib/test/test_pyrepl/test_eventqueue.py | 11 +- Lib/test/test_pyrepl/test_pyrepl.py | 9 +- Lib/test/test_pyrepl/test_terminfo.py | 651 ++++++++++++++++++ Lib/test/test_pyrepl/test_unix_console.py | 40 +- ...-07-18-17-15-00.gh-issue-135621.9cyCNb.rst | 2 + 11 files changed, 1230 insertions(+), 161 deletions(-) delete mode 100644 Lib/_pyrepl/_minimal_curses.py delete mode 100644 Lib/_pyrepl/curses.py create mode 100644 Lib/_pyrepl/terminfo.py create mode 100644 Lib/test/test_pyrepl/test_terminfo.py create mode 100644 Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50ac7..00000000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f6835d1..00000000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 00000000000000..063a285bb9900c --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,530 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on +_STRING_CAPABILITY_NAMES = {name: i for i, name in enumerate(_STRING_NAMES)} + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _names: list[str] = field(default_factory=list) + _booleans: list[int] = field(default_factory=list) + _numbers: list[int] = field(default_factory=list) + _strings: list[bytes | None] = field(default_factory=list) + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + names = data[offset : offset + name_size - 1].decode( + "ascii", errors="ignore" + ) + offset += name_size + + # Read boolean capabilities + if offset + bool_count > len(data): + raise ValueError(too_short) + booleans = list(data[offset : offset + bool_count]) + offset += bool_count + + # Align to even byte boundary for numbers + if offset % 2: + offset += 1 + + # Read numeric capabilities + numbers = [] + for i in range(num_count): + if offset + number_size > len(data): + raise ValueError(too_short) + num = struct.unpack( + number_format, data[offset : offset + number_size] + )[0] + numbers.append(num) + offset += number_size + + # Read string offsets + string_offsets = [] + for i in range(str_count): + if offset + 2 > len(data): + raise ValueError(too_short) + off = struct.unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + strings: list[bytes | None] = [] + for off in string_offsets: + if off < 0: + strings.append(CANCELLED_STRING) + elif off < len(string_table): + # Find null terminator + end = off + while end < len(string_table) and string_table[end] != 0: + end += 1 + if end <= len(string_table): + strings.append(string_table[off:end]) + else: + strings.append(ABSENT_STRING) + else: + strings.append(ABSENT_STRING) + + self._names = names.split("|") + self._booleans = booleans + self._numbers = numbers + self._strings = strings + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_ti.c:tigetstr() + + The ncurses version searches through compiled terminfo data structures. + This version first checks parsed terminfo data, then falls back to + hardcoded capabilities. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + if self._capabilities: + # Fallbacks populated, use them + return self._capabilities.get(cap) + + # Look up in standard capabilities first + if cap in _STRING_CAPABILITY_NAMES: + index = _STRING_CAPABILITY_NAMES[cap] + if index < len(self._strings): + return self._strings[index] + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + return None + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index d21cdd9b076d86..a7e49923191c07 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,7 +33,7 @@ import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event from .fancy_termios import tcgetattr, tcsetattr from .trace import trace @@ -60,7 +60,7 @@ class InvalidTerminal(RuntimeError): pass -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) SIGWINCH_EVENT = "repaint" @@ -157,7 +157,7 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term @overload @@ -167,7 +167,7 @@ def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -201,7 +201,7 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) + self.event_queue = EventQueue(self.input_fd, self.encoding, self.terminfo) self.cursor_visible = 1 signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -597,14 +597,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -701,7 +701,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 29b3e9dd5efd07..2a9cca59e7477f 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,7 +18,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from . import curses +from .terminfo import TermInfo from .trace import trace from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE @@ -54,22 +54,23 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes + class EventQueue(BaseEventQueue): - def __init__(self, fd: int, encoding: str) -> None: - keycodes = get_terminal_keycodes() + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] keycodes[backspace] = "backspace" diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index 8359d9844623c2..8ef472eb0cffaf 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -1,14 +1,14 @@ import os -import sys -from test.support import requires, load_package_tests -from test.support.import_helper import import_module - -if sys.platform != "win32": - # On non-Windows platforms, testing pyrepl currently requires that the - # 'curses' resource be given on the regrtest command line using the -u - # option. Additionally, we need to attempt to import curses and readline. - requires("curses") - curses = import_module("curses") +from test.support import load_package_tests +import unittest + + +try: + import termios +except ImportError: + raise unittest.SkipTest("termios required") +else: + del termios def load_tests(*args): diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index edfe6ac4748f33..69d9612b70dc77 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -3,6 +3,8 @@ from unittest.mock import patch from test import support +from _pyrepl import terminfo + try: from _pyrepl.console import Event from _pyrepl import base_eventqueue @@ -172,17 +174,22 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "a")) +class EmptyTermInfo(terminfo.TermInfo): + def get(self, cap: str) -> bytes: + return b"" + + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): def setUp(self): - self.enterContext(patch("_pyrepl.curses.tigetstr", lambda x: b"")) self.file = tempfile.TemporaryFile() def tearDown(self) -> None: self.file.close() def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: - return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8") + ti = EmptyTermInfo("ansi") + return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8", ti) @unittest.skipUnless(support.MS_WINDOWS, "No Windows event queue on Unix") diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 98bae7dd703fd9..de10a8a07c8f3f 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -9,10 +9,10 @@ import sys import tempfile from pkgutil import ModuleInfo -from unittest import TestCase, skipUnless, skipIf +from unittest import TestCase, skipUnless, skipIf, SkipTest from unittest.mock import patch from test.support import force_not_colorized, make_clean_env, Py_DEBUG -from test.support import SHORT_TIMEOUT, STDLIB_DIR +from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -38,6 +38,10 @@ class ReplTestCase(TestCase): + def setUp(self): + if not has_subprocess_support: + raise SkipTest("test module requires subprocess") + def run_repl( self, repl_input: str | list[str], @@ -1371,6 +1375,7 @@ def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be # added later in test methods to patched os.environ. + super().setUp() patcher = patch('os.environ', new=make_clean_env()) self.addCleanup(patcher.stop) patcher.start() diff --git a/Lib/test/test_pyrepl/test_terminfo.py b/Lib/test/test_pyrepl/test_terminfo.py new file mode 100644 index 00000000000000..562cf5c905bd67 --- /dev/null +++ b/Lib/test/test_pyrepl/test_terminfo.py @@ -0,0 +1,651 @@ +"""Tests comparing PyREPL's pure Python curses implementation with the standard curses module.""" + +import json +import os +import subprocess +import sys +import unittest +from test.support import requires, has_subprocess_support +from textwrap import dedent + +# Only run these tests if curses is available +requires("curses") + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + _curses = None + +from _pyrepl import terminfo + + +ABSENT_STRING = terminfo.ABSENT_STRING +CANCELLED_STRING = terminfo.CANCELLED_STRING + + +class TestCursesCompatibility(unittest.TestCase): + """Test that PyREPL's curses implementation matches the standard curses behavior. + + Python's `curses` doesn't allow calling `setupterm()` again with a different + $TERM in the same process, so we subprocess all `curses` tests to get correctly + set up terminfo.""" + + @classmethod + def setUpClass(cls): + if _curses is None: + raise unittest.SkipTest( + "`curses` capability provided to regrtest but `_curses` not importable" + ) + + if not has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + # we need to ensure there's a terminfo database on the system and that + # `infocmp` works + cls.infocmp("dumb") + + def setUp(self): + self.original_term = os.environ.get("TERM", None) + + def tearDown(self): + if self.original_term is not None: + os.environ["TERM"] = self.original_term + elif "TERM" in os.environ: + del os.environ["TERM"] + + @classmethod + def infocmp(cls, term) -> list[str]: + all_caps = [] + try: + result = subprocess.run( + ["infocmp", "-l1", term], + capture_output=True, + text=True, + check=True, + ) + except Exception: + raise unittest.SkipTest("calling `infocmp` failed on the system") + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("#"): + if "terminfo" not in line and "termcap" in line: + # PyREPL terminfo doesn't parse termcap databases + raise unittest.SkipTest( + "curses using termcap.db: no terminfo database on" + " the system" + ) + elif "=" in line: + cap_name = line.split("=")[0] + all_caps.append(cap_name) + + return all_caps + + def test_setupterm_basic(self): + """Test basic setupterm functionality.""" + # Test with explicit terminal type + test_terms = ["xterm", "xterm-256color", "vt100", "ansi"] + + for term in test_terms: + with self.subTest(term=term): + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(term)}, 1) + print(json.dumps({{"success": True}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + # Set up with PyREPL curses + try: + terminfo.TermInfo(term, fallback=False) + pyrepl_success = True + except Exception as e: + pyrepl_success = False + pyrepl_error = e + + # Both should succeed or both should fail + if std_success: + self.assertTrue( + pyrepl_success, + f"Standard curses succeeded but PyREPL failed for {term}", + ) + else: + # If standard curses failed, PyREPL might still succeed with fallback + # This is acceptable as PyREPL has hardcoded fallbacks + pass + + def test_setupterm_none(self): + """Test setupterm with None (uses TERM from environment).""" + # Test with current TERM + ncurses_code = dedent( + """ + import _curses + import json + try: + _curses.setupterm(None, 1) + print(json.dumps({"success": True})) + except Exception as e: + print(json.dumps({"success": False, "error": str(e)})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + try: + terminfo.TermInfo(None, fallback=False) + pyrepl_success = True + except Exception: + pyrepl_success = False + + # Both should have same result + if std_success: + self.assertTrue( + pyrepl_success, + "Standard curses succeeded but PyREPL failed for None", + ) + + def test_tigetstr_common_capabilities(self): + """Test tigetstr for common terminal capabilities.""" + # Test with a known terminal type + term = "xterm" + + # Get ALL capabilities from infocmp + all_caps = self.infocmp(term) + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + + ncurses_data = json.loads(result.stdout) + + ti = terminfo.TermInfo(term, fallback=False) + + # Test every single capability + for cap in all_caps: + if cap not in ncurses_data or ncurses_data[cap] == "error": + continue + + with self.subTest(capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap}: ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_tigetstr_input_types(self): + """Test tigetstr with different input types.""" + term = "xterm" + cap = "cup" + + # Test standard curses behavior with string in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with string input + try: + std_str_result = _curses.tigetstr({repr(cap)}) + std_accepts_str = True + if std_str_result is None: + std_str_val = None + elif std_str_result == -1: + std_str_val = -1 + else: + std_str_val = list(std_str_result) + except TypeError: + std_accepts_str = False + std_str_val = None + + print(json.dumps({{ + "accepts_str": std_accepts_str, + "str_result": std_str_val + }})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # PyREPL behavior with string + try: + pyrepl_str_result = ti.get(cap) + pyrepl_accepts_str = True + except TypeError: + pyrepl_accepts_str = False + + # PyREPL should also only accept strings for compatibility + with self.assertRaises(TypeError): + ti.get(cap.encode("ascii")) + + # Both should accept string input + self.assertEqual( + pyrepl_accepts_str, + ncurses_data["accepts_str"], + "PyREPL and standard curses should have same string handling", + ) + self.assertTrue( + pyrepl_accepts_str, "PyREPL should accept string input" + ) + + def test_tparm_basic(self): + """Test basic tparm functionality.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test cursor positioning (cup) + cup = ti.get("cup") + if cup and cup not in {ABSENT_STRING, CANCELLED_STRING}: + # Test various parameter combinations + test_cases = [ + (0, 0), # Top-left + (5, 10), # Arbitrary position + (23, 79), # Bottom-right of standard terminal + (999, 999), # Large values + ] + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Get cup capability + cup = _curses.tigetstr('cup') + results = {{}} + + for row, col in {repr(test_cases)}: + try: + result = _curses.tparm(cup, row, col) + results[f"{{row}},{{col}}"] = list(result) + except Exception as e: + results[f"{{row}},{{col}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for row, col in test_cases: + with self.subTest(row=row, col=col): + # Standard curses tparm from subprocess + key = f"{row},{col}" + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + # PyREPL curses tparm + pyrepl_result = terminfo.tparm(cup, row, col) + + # Results should be identical + self.assertEqual( + pyrepl_result, + std_result, + f"tparm(cup, {row}, {col}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + else: + raise unittest.SkipTest( + "test_tparm_basic() requires the `cup` capability" + ) + + def test_tparm_multiple_params(self): + """Test tparm with capabilities using multiple parameters.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test capabilities that take parameters + param_caps = { + "cub": 1, # cursor_left with count + "cuf": 1, # cursor_right with count + "cuu": 1, # cursor_up with count + "cud": 1, # cursor_down with count + "dch": 1, # delete_character with count + "ich": 1, # insert_character with count + } + + # Get all capabilities from PyREPL first + pyrepl_caps = {} + for cap in param_caps: + cap_value = ti.get(cap) + if cap_value and cap_value not in { + ABSENT_STRING, + CANCELLED_STRING, + }: + pyrepl_caps[cap] = cap_value + + if not pyrepl_caps: + self.skipTest("No parametrized capabilities found") + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + param_caps = {repr(param_caps)} + test_values = [1, 5, 10, 99] + results = {{}} + + for cap in param_caps: + cap_value = _curses.tigetstr(cap) + if cap_value and cap_value != -1: + for value in test_values: + try: + result = _curses.tparm(cap_value, value) + results[f"{{cap}},{{value}}"] = list(result) + except Exception as e: + results[f"{{cap}},{{value}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for cap, cap_value in pyrepl_caps.items(): + with self.subTest(capability=cap): + # Test with different parameter values + for value in [1, 5, 10, 99]: + key = f"{cap},{value}" + if key in ncurses_data: + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + pyrepl_result = terminfo.tparm(cap_value, value) + self.assertEqual( + pyrepl_result, + std_result, + f"tparm({cap}, {value}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + + def test_tparm_null_handling(self): + """Test tparm with None/null input.""" + term = "xterm" + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with None + try: + _curses.tparm(None) + raises_typeerror = False + except TypeError: + raises_typeerror = True + except Exception as e: + raises_typeerror = False + error_type = type(e).__name__ + + print(json.dumps({{"raises_typeerror": raises_typeerror}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # Test with None - both should raise TypeError + if ncurses_data["raises_typeerror"]: + with self.assertRaises(TypeError): + terminfo.tparm(None) + else: + # If ncurses doesn't raise TypeError, PyREPL shouldn't either + try: + terminfo.tparm(None) + except TypeError: + self.fail("PyREPL raised TypeError but ncurses did not") + + def test_special_terminals(self): + """Test with special terminal types.""" + special_terms = [ + "dumb", # Minimal terminal + "unknown", # Should fall back to defaults + "linux", # Linux console + "screen", # GNU Screen + "tmux", # tmux + ] + + # Get all string capabilities from ncurses + for term in special_terms: + with self.subTest(term=term): + all_caps = self.infocmp(term) + ncurses_code = dedent( + f""" + import _curses + import json + import sys + + try: + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + # Convert bytes to list of ints for JSON + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + except Exception as e: + print(json.dumps({{"error": str(e)}})) + """ + ) + + # Get ncurses results + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"Failed to get ncurses data for {term}: {result.stderr}" + ) + + try: + ncurses_data = json.loads(result.stdout) + except json.JSONDecodeError: + self.fail( + f"Failed to parse ncurses output for {term}: {result.stdout}" + ) + + if "error" in ncurses_data and len(ncurses_data) == 1: + # ncurses failed to setup this terminal + # PyREPL should still work with fallback + ti = terminfo.TermInfo(term, fallback=True) + continue + + ti = terminfo.TermInfo(term, fallback=False) + + # Compare all capabilities + for cap in all_caps: + if cap not in ncurses_data: + continue + + with self.subTest(term=term, capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + # Convert back to bytes + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + # Both should return the same value + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap} for {term}: " + f"ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_terminfo_fallback(self): + """Test that PyREPL falls back gracefully when terminfo is not found.""" + # Use a non-existent terminal type + fake_term = "nonexistent-terminal-type-12345" + + # Check if standard curses can setup this terminal in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(fake_term)}, 1) + print(json.dumps({{"success": True}})) + except _curses.error: + print(json.dumps({{"success": False, "error": "curses.error"}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + if ncurses_data["success"]: + # If it succeeded, skip this test as we can't test fallback + self.skipTest( + f"System unexpectedly has terminfo for '{fake_term}'" + ) + + # PyREPL should succeed with fallback + try: + ti = terminfo.TermInfo(fake_term, fallback=True) + pyrepl_ok = True + except Exception: + pyrepl_ok = False + + self.assertTrue( + pyrepl_ok, "PyREPL should fall back for unknown terminals" + ) + + # Should still be able to get basic capabilities + bel = ti.get("bel") + self.assertIsNotNone( + bel, "PyREPL should provide basic capabilities after fallback" + ) + + def test_invalid_terminal_names(self): + cases = [ + (42, TypeError), + ("", ValueError), + ("w\x00t", ValueError), + (f"..{os.sep}name", ValueError), + ] + + for term, exc in cases: + with self.subTest(term=term): + with self.assertRaises(exc): + terminfo._validate_terminal_name_or_raise(term) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index b3f7dc028fe210..ab1236768cfb3e 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -16,9 +16,13 @@ except ImportError: pass +from _pyrepl.terminfo import _TERMINAL_CAPABILITIES + +TERM_CAPABILITIES = _TERMINAL_CAPABILITIES["ansi"] + def unix_console(events, **kwargs): - console = UnixConsole() + console = UnixConsole(term="xterm") console.get_event = MagicMock(side_effect=events) console.getpending = MagicMock(return_value=Event("key", "")) @@ -50,41 +54,11 @@ def unix_console(events, **kwargs): ) -TERM_CAPABILITIES = { - "bel": b"\x07", - "civis": b"\x1b[?25l", - "clear": b"\x1b[H\x1b[2J", - "cnorm": b"\x1b[?12l\x1b[?25h", - "cub": b"\x1b[%p1%dD", - "cub1": b"\x08", - "cud": b"\x1b[%p1%dB", - "cud1": b"\n", - "cuf": b"\x1b[%p1%dC", - "cuf1": b"\x1b[C", - "cup": b"\x1b[%i%p1%d;%p2%dH", - "cuu": b"\x1b[%p1%dA", - "cuu1": b"\x1b[A", - "dch1": b"\x1b[P", - "dch": b"\x1b[%p1%dP", - "el": b"\x1b[K", - "hpa": b"\x1b[%i%p1%dG", - "ich": b"\x1b[%p1%d@", - "ich1": None, - "ind": b"\n", - "pad": None, - "ri": b"\x1bM", - "rmkx": b"\x1b[?1l\x1b>", - "smkx": b"\x1b[?1h\x1b=", -} - - @unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") -@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) @patch( - "_pyrepl.curses.tparm", + "_pyrepl.terminfo.tparm", lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), ) -@patch("_pyrepl.curses.setupterm", lambda a, b: None) @patch( "termios.tcgetattr", lambda _: [ @@ -321,7 +295,7 @@ def same_console(events): def test_getheightwidth_with_invalid_environ(self, _os_write): # gh-128636 - console = UnixConsole() + console = UnixConsole(term="xterm") with os_helper.EnvironmentVarGuard() as env: env["LINES"] = "" self.assertIsInstance(console.getheightwidth(), tuple) diff --git a/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst b/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst new file mode 100644 index 00000000000000..fe7f962ccbb096 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst @@ -0,0 +1,2 @@ +PyREPL no longer depends on the :mod:`curses` standard library. Contributed +by Łukasz Langa. From 0d87bb66d3c2fc6331c3b1f9d649ebc9bb385c5a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 14:29:51 +0200 Subject: [PATCH 683/713] [3.14] gh-121028: Soft-deprecate sys.api_version (GH-136463) (GH-136928) (cherry picked from commit 658599c15d13ee3a5cb56c3d9fccaa195465d4b5) Co-authored-by: Petr Viktorin --- Doc/library/sys.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 05bc7cfb9dc089..52f0af31c68726 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -2191,8 +2191,11 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: api_version - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. + The C API version, equivalent to the C macro :c:macro:`PYTHON_API_VERSION`. + Defined for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. .. data:: version_info From af8d1f56f5147f685d0a79f3964fdb5bb07746c2 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 21 Jul 2025 17:34:54 +0200 Subject: [PATCH 684/713] [3.14] gh-132661: Disallow `Template`/`str` concatenation after PEP 750 spec update (#135996) (#136901) Co-authored-by: Dave Peck Co-authored-by: sobolevn --- Grammar/python.gram | 11 +- Lib/_ast_unparse.py | 26 +- Lib/test/test_ast/test_ast.py | 7 - Lib/test/test_tstring.py | 73 +- Lib/test/test_unparse.py | 4 - ...-07-08-23-22-08.gh-issue-132661.34ftJl.rst | 5 + Objects/templateobject.c | 91 +- Objects/unicodeobject.c | 15 +- Parser/action_helpers.c | 17 +- Parser/parser.c | 2502 +++++++++-------- Parser/pegen.h | 1 + Python/ast_unparse.c | 86 +- Python/codegen.c | 10 - 13 files changed, 1406 insertions(+), 1442 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst diff --git a/Grammar/python.gram b/Grammar/python.gram index 8b15cf29fa02e2..24017c0023c109 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -985,7 +985,10 @@ tstring[expr_ty] (memo): _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) } string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) } -strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } +strings[expr_ty] (memo): + | invalid_string_tstring_concat + | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } + | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } @@ -1546,6 +1549,12 @@ invalid_tstring_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") } +invalid_string_tstring_concat: + | a=(fstring|string)+ b[expr_ty]=tstring { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + | a=tstring+ b[expr_ty]=(fstring|string) { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index c25066eb107de1..16cf56f62cc1e5 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -626,35 +626,11 @@ def _write_ftstring(self, values, prefix): ) self._ftstring_helper(fstring_parts) - def _tstring_helper(self, node): - if not node.values: - self._write_ftstring([], "t") - return - last_idx = 0 - for i, value in enumerate(node.values): - # This can happen if we have an implicit concat of a t-string - # with an f-string - if isinstance(value, FormattedValue): - if i > last_idx: - # Write t-string until here - self._write_ftstring(node.values[last_idx:i], "t") - self.write(" ") - # Write f-string with the current formatted value - self._write_ftstring([node.values[i]], "f") - if i + 1 < len(node.values): - # Only add a space if there are more values after this - self.write(" ") - last_idx = i + 1 - - if last_idx < len(node.values): - # Write t-string from last_idx to end - self._write_ftstring(node.values[last_idx:], "t") - def visit_JoinedStr(self, node): self._write_ftstring(node.values, "f") def visit_TemplateStr(self, node): - self._tstring_helper(node) + self._write_ftstring(node.values, "t") def _write_ftstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 56e170e128b48c..269f3ff84f6ab1 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -999,13 +999,6 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - # Test AST for implicit concat of t-string with f-string - tree = ast.parse('t"Hello {name}" f"{name}"') - self.assertIsInstance(tree.body[0].value, ast.TemplateStr) - self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) - self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue) - class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index aabae38556735b..74653c77c55de1 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -150,7 +150,6 @@ def test_raw_tstrings(self): t = tr"{path}\Documents" self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) - def test_template_concatenation(self): # Test template + template t1 = t"Hello, " @@ -161,9 +160,10 @@ def test_template_concatenation(self): # Test template + string t1 = t"Hello" - combined = t1 + ", world" - self.assertTStringEqual(combined, ("Hello, world",), ()) - self.assertEqual(fstring(combined), "Hello, world") + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" # Test template + template with interpolation name = "Python" @@ -174,9 +174,10 @@ def test_template_concatenation(self): self.assertEqual(fstring(combined), "Hello, Python") # Test string + template - t = "Hello, " + t"{name}" - self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) - self.assertEqual(fstring(t), "Hello, Python") + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" def test_nested_templates(self): # Test a template inside another template expression @@ -241,52 +242,28 @@ def test_literal_concatenation(self): self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) self.assertEqual(fstring(t), "Hello, Python") - # Test concatenation with string literal - name = "Python" - t = t"Hello, {name}" "and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with Unicode literal - name = "Python" - t = t"Hello, {name}" u"and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with f-string literal - tab = '\t' - t = t"Tab: {tab}. " f"f-tab: {tab}." - self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")]) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.") - - # Test concatenation with raw string literal - tab = '\t' - t = t"Tab: {tab}. " r"Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.") - - # Test concatenation with raw f-string literal - tab = '\t' - t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.") - + # Test disallowed mix of t-string and string/f-string (incl. bytes) what = 't' - expected_msg = 'cannot mix bytes and nonbytes literals' + expected_msg = 'cannot mix t-string literals with string or bytes literals' for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", "t'{what}-string literal' b'bytes literal'", "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", ): - with self.assertRaisesRegex(SyntaxError, expected_msg): - eval(case) + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) def test_triple_quoted(self): # Test triple-quoted t-strings diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d4db5e60af7978..0d6b05bc660b76 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -206,10 +206,6 @@ def test_tstrings(self): self.check_ast_roundtrip("t'foo'") self.check_ast_roundtrip("t'foo {bar}'") self.check_ast_roundtrip("t'foo {bar!s:.2f}'") - self.check_ast_roundtrip("t'foo {bar}' f'{bar}'") - self.check_ast_roundtrip("f'{bar}' t'foo {bar}'") - self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'") - self.check_ast_roundtrip("t'foo {bar}' u'bar'") def test_strings(self): self.check_ast_roundtrip("u'foo'") diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst new file mode 100644 index 00000000000000..5d59e024389c18 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst @@ -0,0 +1,5 @@ +Reflect recent :pep:`750` change. + +Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. +Also, disallow implicit concatenation of t-string literals with string or +f-string literals. diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 4293a311c440f7..ac38e4de435d5d 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -30,7 +30,8 @@ templateiter_next(PyObject *op) Py_SETREF(item, PyIter_Next(self->interpolationsiter)); self->from_strings = 1; } - } else { + } + else { item = PyIter_Next(self->interpolationsiter); self->from_strings = 1; } @@ -245,54 +246,6 @@ template_iter(PyObject *op) return (PyObject *)iter; } -static PyObject * -template_strings_append_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1); - PyObject *concat = PyUnicode_Concat(string, str); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - for (Py_ssize_t i = 0; i < stringslen - 1; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - PyTuple_SET_ITEM(newstrings, stringslen - 1, concat); - - return newstrings; -} - -static PyObject * -template_strings_prepend_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, 0); - PyObject *concat = PyUnicode_Concat(str, string); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - PyTuple_SET_ITEM(newstrings, 0, concat); - for (Py_ssize_t i = 1; i < stringslen; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - - return newstrings; -} - static PyObject * template_strings_concat(PyObject *left, PyObject *right) { @@ -344,47 +297,17 @@ template_concat_templates(templateobject *self, templateobject *other) return newtemplate; } -static PyObject * -template_concat_template_str(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_append_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - -static PyObject * -template_concat_str_template(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_prepend_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - PyObject * _PyTemplate_Concat(PyObject *self, PyObject *other) { if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) { return template_concat_templates((templateobject *) self, (templateobject *) other); } - else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) { - return template_concat_template_str((templateobject *) self, other); - } - else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) { - return template_concat_str_template((templateobject *) other, self); - } - else { - Py_RETURN_NOTIMPLEMENTED; - } + + PyErr_Format(PyExc_TypeError, + "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template", + other); + return NULL; } static PyObject * diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 1d01dad9e33e29..c7f560adb99356 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_template.h" // _PyTemplate_Concat() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -11639,16 +11638,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right) return NULL; if (!PyUnicode_Check(right)) { - if (_PyTemplate_CheckExact(right)) { - // str + tstring is implemented in the tstring type - return _PyTemplate_Concat(left, right); - } - else { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - Py_TYPE(right)->tp_name); - return NULL; - } + PyErr_Format(PyExc_TypeError, + "can only concatenate str (not \"%.200s\") to str", + Py_TYPE(right)->tp_name); + return NULL; } /* Shortcuts */ diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 0d362bf7a9111a..ebc94715b6f361 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -1834,8 +1834,8 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings, return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena); } -static expr_ty -_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings, +expr_ty +_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { @@ -1853,7 +1853,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t len = asdl_seq_LEN(strings); assert(len > 0); - int t_string_found = 0; int f_string_found = 0; int unicode_string_found = 0; int bytes_found = 0; @@ -1873,7 +1872,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, f_string_found = 1; break; case TemplateStr_kind: - t_string_found = 1; + // python.gram handles this; we should never get here + assert(0); break; default: f_string_found = 1; @@ -1882,13 +1882,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } // Cannot mix unicode and bytes - if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) { + if ((unicode_string_found || f_string_found) && bytes_found) { RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals"); return NULL; } // If it's only bytes or only unicode string, do a simple concat - if (!f_string_found && !t_string_found) { + if (!f_string_found) { if (len == 1) { return asdl_seq_GET(strings, 0); } @@ -1902,11 +1902,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } } - if (t_string_found) { - return _build_concatenated_template_str(p, strings, lineno, - col_offset, end_lineno, end_col_offset, arena); - } - return _build_concatenated_joined_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); } diff --git a/Parser/parser.c b/Parser/parser.c index 932ad0f594124a..08370f51af480f 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -348,185 +348,187 @@ static char *soft_keywords[] = { #define invalid_fstring_conversion_character_type 1261 #define invalid_tstring_replacement_field_type 1262 #define invalid_tstring_conversion_character_type 1263 -#define invalid_arithmetic_type 1264 -#define invalid_factor_type 1265 -#define invalid_type_params_type 1266 -#define _loop0_1_type 1267 -#define _loop1_2_type 1268 -#define _loop0_3_type 1269 -#define _gather_4_type 1270 -#define _tmp_5_type 1271 -#define _tmp_6_type 1272 -#define _tmp_7_type 1273 -#define _tmp_8_type 1274 -#define _tmp_9_type 1275 -#define _tmp_10_type 1276 -#define _tmp_11_type 1277 -#define _loop1_12_type 1278 -#define _tmp_13_type 1279 -#define _loop0_14_type 1280 -#define _gather_15_type 1281 -#define _tmp_16_type 1282 -#define _tmp_17_type 1283 -#define _loop0_18_type 1284 -#define _loop1_19_type 1285 -#define _loop0_20_type 1286 -#define _gather_21_type 1287 -#define _tmp_22_type 1288 -#define _loop0_23_type 1289 -#define _gather_24_type 1290 -#define _loop1_25_type 1291 -#define _tmp_26_type 1292 -#define _tmp_27_type 1293 -#define _loop0_28_type 1294 -#define _loop0_29_type 1295 -#define _loop1_30_type 1296 -#define _loop1_31_type 1297 -#define _loop0_32_type 1298 -#define _loop1_33_type 1299 -#define _loop0_34_type 1300 -#define _gather_35_type 1301 -#define _tmp_36_type 1302 -#define _loop1_37_type 1303 -#define _loop1_38_type 1304 -#define _loop1_39_type 1305 -#define _loop0_40_type 1306 -#define _gather_41_type 1307 -#define _tmp_42_type 1308 -#define _tmp_43_type 1309 -#define _tmp_44_type 1310 -#define _loop0_45_type 1311 -#define _gather_46_type 1312 -#define _loop0_47_type 1313 -#define _gather_48_type 1314 -#define _tmp_49_type 1315 -#define _loop0_50_type 1316 -#define _gather_51_type 1317 -#define _loop0_52_type 1318 -#define _gather_53_type 1319 -#define _loop0_54_type 1320 -#define _gather_55_type 1321 -#define _loop1_56_type 1322 -#define _loop1_57_type 1323 -#define _loop0_58_type 1324 -#define _gather_59_type 1325 -#define _loop1_60_type 1326 -#define _loop1_61_type 1327 -#define _loop1_62_type 1328 -#define _tmp_63_type 1329 -#define _loop0_64_type 1330 -#define _gather_65_type 1331 -#define _tmp_66_type 1332 -#define _tmp_67_type 1333 -#define _tmp_68_type 1334 -#define _tmp_69_type 1335 -#define _tmp_70_type 1336 -#define _loop0_71_type 1337 -#define _loop0_72_type 1338 -#define _loop1_73_type 1339 -#define _loop1_74_type 1340 -#define _loop0_75_type 1341 -#define _loop1_76_type 1342 -#define _loop0_77_type 1343 -#define _loop0_78_type 1344 -#define _loop0_79_type 1345 -#define _loop0_80_type 1346 -#define _loop1_81_type 1347 -#define _tmp_82_type 1348 -#define _loop0_83_type 1349 -#define _gather_84_type 1350 -#define _loop1_85_type 1351 -#define _loop0_86_type 1352 -#define _tmp_87_type 1353 -#define _loop0_88_type 1354 -#define _gather_89_type 1355 -#define _tmp_90_type 1356 -#define _loop0_91_type 1357 -#define _gather_92_type 1358 -#define _loop0_93_type 1359 -#define _gather_94_type 1360 -#define _loop0_95_type 1361 -#define _loop0_96_type 1362 -#define _gather_97_type 1363 -#define _loop1_98_type 1364 -#define _tmp_99_type 1365 -#define _loop0_100_type 1366 -#define _gather_101_type 1367 -#define _loop0_102_type 1368 -#define _gather_103_type 1369 -#define _tmp_104_type 1370 -#define _tmp_105_type 1371 -#define _loop0_106_type 1372 -#define _gather_107_type 1373 -#define _tmp_108_type 1374 -#define _tmp_109_type 1375 -#define _tmp_110_type 1376 -#define _tmp_111_type 1377 -#define _tmp_112_type 1378 -#define _loop1_113_type 1379 -#define _tmp_114_type 1380 -#define _tmp_115_type 1381 -#define _tmp_116_type 1382 -#define _tmp_117_type 1383 -#define _tmp_118_type 1384 -#define _loop0_119_type 1385 -#define _loop0_120_type 1386 -#define _tmp_121_type 1387 -#define _tmp_122_type 1388 -#define _tmp_123_type 1389 -#define _tmp_124_type 1390 -#define _tmp_125_type 1391 -#define _tmp_126_type 1392 -#define _tmp_127_type 1393 -#define _tmp_128_type 1394 -#define _tmp_129_type 1395 -#define _loop0_130_type 1396 -#define _gather_131_type 1397 -#define _tmp_132_type 1398 -#define _tmp_133_type 1399 -#define _tmp_134_type 1400 -#define _tmp_135_type 1401 -#define _loop0_136_type 1402 -#define _gather_137_type 1403 -#define _tmp_138_type 1404 -#define _loop0_139_type 1405 -#define _gather_140_type 1406 -#define _loop0_141_type 1407 -#define _gather_142_type 1408 -#define _tmp_143_type 1409 -#define _loop0_144_type 1410 -#define _tmp_145_type 1411 -#define _tmp_146_type 1412 -#define _tmp_147_type 1413 -#define _tmp_148_type 1414 -#define _tmp_149_type 1415 -#define _tmp_150_type 1416 -#define _tmp_151_type 1417 -#define _tmp_152_type 1418 -#define _tmp_153_type 1419 -#define _tmp_154_type 1420 -#define _tmp_155_type 1421 -#define _tmp_156_type 1422 -#define _tmp_157_type 1423 -#define _tmp_158_type 1424 -#define _tmp_159_type 1425 -#define _tmp_160_type 1426 -#define _tmp_161_type 1427 -#define _tmp_162_type 1428 -#define _tmp_163_type 1429 -#define _tmp_164_type 1430 -#define _tmp_165_type 1431 -#define _tmp_166_type 1432 -#define _tmp_167_type 1433 -#define _tmp_168_type 1434 -#define _tmp_169_type 1435 -#define _tmp_170_type 1436 -#define _loop0_171_type 1437 -#define _tmp_172_type 1438 -#define _tmp_173_type 1439 -#define _tmp_174_type 1440 -#define _tmp_175_type 1441 -#define _tmp_176_type 1442 +#define invalid_string_tstring_concat_type 1264 +#define invalid_arithmetic_type 1265 +#define invalid_factor_type 1266 +#define invalid_type_params_type 1267 +#define _loop0_1_type 1268 +#define _loop1_2_type 1269 +#define _loop0_3_type 1270 +#define _gather_4_type 1271 +#define _tmp_5_type 1272 +#define _tmp_6_type 1273 +#define _tmp_7_type 1274 +#define _tmp_8_type 1275 +#define _tmp_9_type 1276 +#define _tmp_10_type 1277 +#define _tmp_11_type 1278 +#define _loop1_12_type 1279 +#define _tmp_13_type 1280 +#define _loop0_14_type 1281 +#define _gather_15_type 1282 +#define _tmp_16_type 1283 +#define _tmp_17_type 1284 +#define _loop0_18_type 1285 +#define _loop1_19_type 1286 +#define _loop0_20_type 1287 +#define _gather_21_type 1288 +#define _tmp_22_type 1289 +#define _loop0_23_type 1290 +#define _gather_24_type 1291 +#define _loop1_25_type 1292 +#define _tmp_26_type 1293 +#define _tmp_27_type 1294 +#define _loop0_28_type 1295 +#define _loop0_29_type 1296 +#define _loop1_30_type 1297 +#define _loop1_31_type 1298 +#define _loop0_32_type 1299 +#define _loop1_33_type 1300 +#define _loop0_34_type 1301 +#define _gather_35_type 1302 +#define _tmp_36_type 1303 +#define _loop1_37_type 1304 +#define _loop1_38_type 1305 +#define _loop1_39_type 1306 +#define _loop0_40_type 1307 +#define _gather_41_type 1308 +#define _tmp_42_type 1309 +#define _tmp_43_type 1310 +#define _tmp_44_type 1311 +#define _loop0_45_type 1312 +#define _gather_46_type 1313 +#define _loop0_47_type 1314 +#define _gather_48_type 1315 +#define _tmp_49_type 1316 +#define _loop0_50_type 1317 +#define _gather_51_type 1318 +#define _loop0_52_type 1319 +#define _gather_53_type 1320 +#define _loop0_54_type 1321 +#define _gather_55_type 1322 +#define _loop1_56_type 1323 +#define _loop1_57_type 1324 +#define _loop0_58_type 1325 +#define _gather_59_type 1326 +#define _loop1_60_type 1327 +#define _loop1_61_type 1328 +#define _loop1_62_type 1329 +#define _tmp_63_type 1330 +#define _loop0_64_type 1331 +#define _gather_65_type 1332 +#define _tmp_66_type 1333 +#define _tmp_67_type 1334 +#define _tmp_68_type 1335 +#define _tmp_69_type 1336 +#define _tmp_70_type 1337 +#define _loop0_71_type 1338 +#define _loop0_72_type 1339 +#define _loop1_73_type 1340 +#define _loop1_74_type 1341 +#define _loop0_75_type 1342 +#define _loop1_76_type 1343 +#define _loop0_77_type 1344 +#define _loop0_78_type 1345 +#define _loop0_79_type 1346 +#define _loop0_80_type 1347 +#define _loop1_81_type 1348 +#define _loop1_82_type 1349 +#define _tmp_83_type 1350 +#define _loop0_84_type 1351 +#define _gather_85_type 1352 +#define _loop1_86_type 1353 +#define _loop0_87_type 1354 +#define _tmp_88_type 1355 +#define _loop0_89_type 1356 +#define _gather_90_type 1357 +#define _tmp_91_type 1358 +#define _loop0_92_type 1359 +#define _gather_93_type 1360 +#define _loop0_94_type 1361 +#define _gather_95_type 1362 +#define _loop0_96_type 1363 +#define _loop0_97_type 1364 +#define _gather_98_type 1365 +#define _loop1_99_type 1366 +#define _tmp_100_type 1367 +#define _loop0_101_type 1368 +#define _gather_102_type 1369 +#define _loop0_103_type 1370 +#define _gather_104_type 1371 +#define _tmp_105_type 1372 +#define _tmp_106_type 1373 +#define _loop0_107_type 1374 +#define _gather_108_type 1375 +#define _tmp_109_type 1376 +#define _tmp_110_type 1377 +#define _tmp_111_type 1378 +#define _tmp_112_type 1379 +#define _tmp_113_type 1380 +#define _loop1_114_type 1381 +#define _tmp_115_type 1382 +#define _tmp_116_type 1383 +#define _tmp_117_type 1384 +#define _tmp_118_type 1385 +#define _tmp_119_type 1386 +#define _loop0_120_type 1387 +#define _loop0_121_type 1388 +#define _tmp_122_type 1389 +#define _tmp_123_type 1390 +#define _tmp_124_type 1391 +#define _tmp_125_type 1392 +#define _tmp_126_type 1393 +#define _tmp_127_type 1394 +#define _tmp_128_type 1395 +#define _tmp_129_type 1396 +#define _tmp_130_type 1397 +#define _loop0_131_type 1398 +#define _gather_132_type 1399 +#define _tmp_133_type 1400 +#define _tmp_134_type 1401 +#define _tmp_135_type 1402 +#define _tmp_136_type 1403 +#define _loop0_137_type 1404 +#define _gather_138_type 1405 +#define _tmp_139_type 1406 +#define _loop0_140_type 1407 +#define _gather_141_type 1408 +#define _loop0_142_type 1409 +#define _gather_143_type 1410 +#define _tmp_144_type 1411 +#define _loop0_145_type 1412 +#define _tmp_146_type 1413 +#define _tmp_147_type 1414 +#define _tmp_148_type 1415 +#define _tmp_149_type 1416 +#define _tmp_150_type 1417 +#define _tmp_151_type 1418 +#define _tmp_152_type 1419 +#define _tmp_153_type 1420 +#define _tmp_154_type 1421 +#define _tmp_155_type 1422 +#define _tmp_156_type 1423 +#define _tmp_157_type 1424 +#define _tmp_158_type 1425 +#define _tmp_159_type 1426 +#define _tmp_160_type 1427 +#define _tmp_161_type 1428 +#define _tmp_162_type 1429 +#define _tmp_163_type 1430 +#define _tmp_164_type 1431 +#define _tmp_165_type 1432 +#define _tmp_166_type 1433 +#define _tmp_167_type 1434 +#define _tmp_168_type 1435 +#define _tmp_169_type 1436 +#define _tmp_170_type 1437 +#define _tmp_171_type 1438 +#define _loop0_172_type 1439 +#define _tmp_173_type 1440 +#define _tmp_174_type 1441 +#define _tmp_175_type 1442 +#define _tmp_176_type 1443 +#define _tmp_177_type 1444 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -792,6 +794,7 @@ static void *invalid_fstring_replacement_field_rule(Parser *p); static void *invalid_fstring_conversion_character_rule(Parser *p); static void *invalid_tstring_replacement_field_rule(Parser *p); static void *invalid_tstring_conversion_character_rule(Parser *p); +static void *invalid_string_tstring_concat_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); @@ -876,46 +879,46 @@ static asdl_seq *_loop0_78_rule(Parser *p); static asdl_seq *_loop0_79_rule(Parser *p); static asdl_seq *_loop0_80_rule(Parser *p); static asdl_seq *_loop1_81_rule(Parser *p); -static void *_tmp_82_rule(Parser *p); -static asdl_seq *_loop0_83_rule(Parser *p); -static asdl_seq *_gather_84_rule(Parser *p); -static asdl_seq *_loop1_85_rule(Parser *p); -static asdl_seq *_loop0_86_rule(Parser *p); -static void *_tmp_87_rule(Parser *p); -static asdl_seq *_loop0_88_rule(Parser *p); -static asdl_seq *_gather_89_rule(Parser *p); -static void *_tmp_90_rule(Parser *p); -static asdl_seq *_loop0_91_rule(Parser *p); -static asdl_seq *_gather_92_rule(Parser *p); -static asdl_seq *_loop0_93_rule(Parser *p); -static asdl_seq *_gather_94_rule(Parser *p); -static asdl_seq *_loop0_95_rule(Parser *p); +static asdl_seq *_loop1_82_rule(Parser *p); +static void *_tmp_83_rule(Parser *p); +static asdl_seq *_loop0_84_rule(Parser *p); +static asdl_seq *_gather_85_rule(Parser *p); +static asdl_seq *_loop1_86_rule(Parser *p); +static asdl_seq *_loop0_87_rule(Parser *p); +static void *_tmp_88_rule(Parser *p); +static asdl_seq *_loop0_89_rule(Parser *p); +static asdl_seq *_gather_90_rule(Parser *p); +static void *_tmp_91_rule(Parser *p); +static asdl_seq *_loop0_92_rule(Parser *p); +static asdl_seq *_gather_93_rule(Parser *p); +static asdl_seq *_loop0_94_rule(Parser *p); +static asdl_seq *_gather_95_rule(Parser *p); static asdl_seq *_loop0_96_rule(Parser *p); -static asdl_seq *_gather_97_rule(Parser *p); -static asdl_seq *_loop1_98_rule(Parser *p); -static void *_tmp_99_rule(Parser *p); -static asdl_seq *_loop0_100_rule(Parser *p); -static asdl_seq *_gather_101_rule(Parser *p); -static asdl_seq *_loop0_102_rule(Parser *p); -static asdl_seq *_gather_103_rule(Parser *p); -static void *_tmp_104_rule(Parser *p); +static asdl_seq *_loop0_97_rule(Parser *p); +static asdl_seq *_gather_98_rule(Parser *p); +static asdl_seq *_loop1_99_rule(Parser *p); +static void *_tmp_100_rule(Parser *p); +static asdl_seq *_loop0_101_rule(Parser *p); +static asdl_seq *_gather_102_rule(Parser *p); +static asdl_seq *_loop0_103_rule(Parser *p); +static asdl_seq *_gather_104_rule(Parser *p); static void *_tmp_105_rule(Parser *p); -static asdl_seq *_loop0_106_rule(Parser *p); -static asdl_seq *_gather_107_rule(Parser *p); -static void *_tmp_108_rule(Parser *p); +static void *_tmp_106_rule(Parser *p); +static asdl_seq *_loop0_107_rule(Parser *p); +static asdl_seq *_gather_108_rule(Parser *p); static void *_tmp_109_rule(Parser *p); static void *_tmp_110_rule(Parser *p); static void *_tmp_111_rule(Parser *p); static void *_tmp_112_rule(Parser *p); -static asdl_seq *_loop1_113_rule(Parser *p); -static void *_tmp_114_rule(Parser *p); +static void *_tmp_113_rule(Parser *p); +static asdl_seq *_loop1_114_rule(Parser *p); static void *_tmp_115_rule(Parser *p); static void *_tmp_116_rule(Parser *p); static void *_tmp_117_rule(Parser *p); static void *_tmp_118_rule(Parser *p); -static asdl_seq *_loop0_119_rule(Parser *p); +static void *_tmp_119_rule(Parser *p); static asdl_seq *_loop0_120_rule(Parser *p); -static void *_tmp_121_rule(Parser *p); +static asdl_seq *_loop0_121_rule(Parser *p); static void *_tmp_122_rule(Parser *p); static void *_tmp_123_rule(Parser *p); static void *_tmp_124_rule(Parser *p); @@ -924,22 +927,22 @@ static void *_tmp_126_rule(Parser *p); static void *_tmp_127_rule(Parser *p); static void *_tmp_128_rule(Parser *p); static void *_tmp_129_rule(Parser *p); -static asdl_seq *_loop0_130_rule(Parser *p); -static asdl_seq *_gather_131_rule(Parser *p); -static void *_tmp_132_rule(Parser *p); +static void *_tmp_130_rule(Parser *p); +static asdl_seq *_loop0_131_rule(Parser *p); +static asdl_seq *_gather_132_rule(Parser *p); static void *_tmp_133_rule(Parser *p); static void *_tmp_134_rule(Parser *p); static void *_tmp_135_rule(Parser *p); -static asdl_seq *_loop0_136_rule(Parser *p); -static asdl_seq *_gather_137_rule(Parser *p); -static void *_tmp_138_rule(Parser *p); -static asdl_seq *_loop0_139_rule(Parser *p); -static asdl_seq *_gather_140_rule(Parser *p); -static asdl_seq *_loop0_141_rule(Parser *p); -static asdl_seq *_gather_142_rule(Parser *p); -static void *_tmp_143_rule(Parser *p); -static asdl_seq *_loop0_144_rule(Parser *p); -static void *_tmp_145_rule(Parser *p); +static void *_tmp_136_rule(Parser *p); +static asdl_seq *_loop0_137_rule(Parser *p); +static asdl_seq *_gather_138_rule(Parser *p); +static void *_tmp_139_rule(Parser *p); +static asdl_seq *_loop0_140_rule(Parser *p); +static asdl_seq *_gather_141_rule(Parser *p); +static asdl_seq *_loop0_142_rule(Parser *p); +static asdl_seq *_gather_143_rule(Parser *p); +static void *_tmp_144_rule(Parser *p); +static asdl_seq *_loop0_145_rule(Parser *p); static void *_tmp_146_rule(Parser *p); static void *_tmp_147_rule(Parser *p); static void *_tmp_148_rule(Parser *p); @@ -965,12 +968,13 @@ static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); static void *_tmp_169_rule(Parser *p); static void *_tmp_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static void *_tmp_172_rule(Parser *p); +static void *_tmp_171_rule(Parser *p); +static asdl_seq *_loop0_172_rule(Parser *p); static void *_tmp_173_rule(Parser *p); static void *_tmp_174_rule(Parser *p); static void *_tmp_175_rule(Parser *p); static void *_tmp_176_rule(Parser *p); +static void *_tmp_177_rule(Parser *p); // file: statements? $ @@ -17057,7 +17061,7 @@ string_rule(Parser *p) return _res; } -// strings: ((fstring | string | tstring))+ +// strings: invalid_string_tstring_concat | ((fstring | string))+ | tstring+ static expr_ty strings_rule(Parser *p) { @@ -17083,18 +17087,37 @@ strings_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // ((fstring | string | tstring))+ + if (p->call_invalid_rules) { // invalid_string_tstring_concat if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + void *invalid_string_tstring_concat_var; + if ( + (invalid_string_tstring_concat_var = invalid_string_tstring_concat_rule(p)) // invalid_string_tstring_concat + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + _res = invalid_string_tstring_concat_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_string_tstring_concat")); + } + { // ((fstring | string))+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string | tstring))+ + (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string))+ ) { - D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -17114,7 +17137,40 @@ strings_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string | tstring))+")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+")); + } + { // tstring+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+")); + asdl_expr_seq* a; + if ( + (a = (asdl_expr_seq*)_loop1_82_rule(p)) // tstring+ + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyPegen_concatenate_tstrings ( p , a , EXTRA ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+")); } _res = NULL; done: @@ -17224,7 +17280,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_82_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_83_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -17439,7 +17495,7 @@ double_starred_kvpairs_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * a; if ( - (a = _gather_84_rule(p)) // ','.double_starred_kvpair+ + (a = _gather_85_rule(p)) // ','.double_starred_kvpair+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -17598,7 +17654,7 @@ for_if_clauses_rule(Parser *p) D(fprintf(stderr, "%*c> for_if_clauses[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); asdl_comprehension_seq* a; if ( - (a = (asdl_comprehension_seq*)_loop1_85_rule(p)) // for_if_clause+ + (a = (asdl_comprehension_seq*)_loop1_86_rule(p)) // for_if_clause+ ) { D(fprintf(stderr, "%*c+ for_if_clauses[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); @@ -17663,7 +17719,7 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_86_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_87_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -17706,7 +17762,7 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_86_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_87_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); @@ -17985,7 +18041,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_87_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_88_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -18234,9 +18290,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_89_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_90_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_90_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_91_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -18326,11 +18382,11 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_92_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_93_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_94_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_95_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); @@ -18352,13 +18408,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_92_var; + asdl_seq * _gather_93_var; if ( - (_gather_92_var = _gather_92_rule(p)) // ','.kwarg_or_starred+ + (_gather_93_var = _gather_93_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_92_var; + _res = _gather_93_var; goto done; } p->mark = _mark; @@ -18371,13 +18427,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_94_var; + asdl_seq * _gather_95_var; if ( - (_gather_94_var = _gather_94_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_95_var = _gather_95_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_94_var; + _res = _gather_95_var; goto done; } p->mark = _mark; @@ -18788,7 +18844,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_95_rule(p)) // ((',' star_target))* + (b = _loop0_96_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18844,7 +18900,7 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_97_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_98_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18894,7 +18950,7 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_98_rule(p)) // ((',' star_target))+ + (b = _loop1_99_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18982,7 +19038,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_99_rule(p)) // !'*' star_target + (a = _tmp_100_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -19905,7 +19961,7 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_101_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_102_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -20263,7 +20319,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20302,7 +20358,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20335,7 +20391,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20455,7 +20511,7 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_103_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_104_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); @@ -20506,7 +20562,7 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, _tmp_104_rule, p) + _PyPegen_lookahead(1, _tmp_105_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); @@ -20592,15 +20648,15 @@ invalid_arguments_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); - asdl_seq * _gather_107_var; - void *_tmp_105_var; + asdl_seq * _gather_108_var; + void *_tmp_106_var; Token * a; if ( - (_tmp_105_var = _tmp_105_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_106_var = _tmp_106_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && (a = _PyPegen_expect_token(p, 12)) // token=',' && - (_gather_107_var = _gather_107_rule(p)) // ','.(starred_expression !'=')+ + (_gather_108_var = _gather_108_rule(p)) // ','.(starred_expression !'=')+ ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); @@ -20634,7 +20690,7 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_108_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_109_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); @@ -20694,13 +20750,13 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_109_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_110_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_110_rule, p) + _PyPegen_lookahead(1, _tmp_111_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); @@ -20838,7 +20894,7 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_111_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_112_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) @@ -20898,7 +20954,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, _tmp_112_rule, p) + _PyPegen_lookahead(0, _tmp_113_rule, p) && (a = expression_rule(p)) // expression && @@ -21246,7 +21302,7 @@ invalid_expression_rule(Parser *p) if ( (string_var = _PyPegen_string_token(p)) // STRING && - (a = _loop1_113_rule(p)) // ((!STRING expression_without_invalid))+ + (a = _loop1_114_rule(p)) // ((!STRING expression_without_invalid))+ && (string_var_1 = _PyPegen_string_token(p)) // STRING ) @@ -21273,7 +21329,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, _tmp_114_rule, p) + _PyPegen_lookahead(0, _tmp_115_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -21309,7 +21365,7 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, _tmp_115_rule, p) + _PyPegen_lookahead(0, _tmp_116_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); @@ -21372,7 +21428,7 @@ invalid_expression_rule(Parser *p) expr_ty b; stmt_ty c; if ( - (a = (stmt_ty)_tmp_116_rule(p)) // pass_stmt | break_stmt | continue_stmt + (a = (stmt_ty)_tmp_117_rule(p)) // pass_stmt | break_stmt | continue_stmt && (_keyword = _PyPegen_expect_token(p, 682)) // token='if' && @@ -21534,7 +21590,7 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); @@ -21560,7 +21616,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, _tmp_118_rule, p) + _PyPegen_lookahead(0, _tmp_119_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -21568,7 +21624,7 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); @@ -21648,7 +21704,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_119_var; + asdl_seq * _loop0_120_var; expr_ty a; expr_ty expression_var; if ( @@ -21656,7 +21712,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_119_var = _loop0_119_rule(p)) // star_named_expressions* + (_loop0_120_var = _loop0_120_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -21713,10 +21769,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_120_var; + asdl_seq * _loop0_121_var; expr_ty a; if ( - (_loop0_120_var = _loop0_120_rule(p)) // ((star_targets '='))* + (_loop0_121_var = _loop0_121_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -21743,10 +21799,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_120_var; + asdl_seq * _loop0_121_var; expr_ty a; if ( - (_loop0_120_var = _loop0_120_rule(p)) // ((star_targets '='))* + (_loop0_121_var = _loop0_121_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -22002,11 +22058,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_121_var; + void *_tmp_122_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_121_var = _tmp_121_rule(p)) // '[' | '(' | '{' + (_tmp_122_var = _tmp_122_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -22033,12 +22089,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_122_var; + void *_tmp_123_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_122_var = _tmp_122_rule(p)) // '[' | '{' + (_tmp_123_var = _tmp_123_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -22068,12 +22124,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_122_var; + void *_tmp_123_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_122_var = _tmp_122_rule(p)) // '[' | '{' + (_tmp_123_var = _tmp_123_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -22209,10 +22265,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); asdl_seq * _loop0_32_var; - void *_tmp_123_var; + void *_tmp_124_var; Token * a; if ( - (_tmp_123_var = _tmp_123_rule(p)) // slash_no_default | slash_with_default + (_tmp_124_var = _tmp_124_rule(p)) // slash_no_default | slash_with_default && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && @@ -22314,16 +22370,16 @@ invalid_parameters_rule(Parser *p) asdl_seq * _loop0_32_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_124_var; + void *_tmp_125_var; Token * a; if ( - (_opt_var = _tmp_123_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_124_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_124_var = _tmp_124_rule(p)) // ',' | param_no_default + (_tmp_125_var = _tmp_125_rule(p)) // ',' | param_no_default && (_loop0_32_var_1 = _loop0_32_rule(p)) // param_maybe_default* && @@ -22402,7 +22458,7 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, _tmp_125_rule, p) + _PyPegen_lookahead(1, _tmp_126_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); @@ -22447,12 +22503,12 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_126_var; + void *_tmp_127_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_126_var = _tmp_126_rule(p)) // ')' | ',' (')' | '**') + (_tmp_127_var = _tmp_127_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); @@ -22536,19 +22592,19 @@ invalid_star_etc_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; asdl_seq * _loop0_32_var; - void *_tmp_127_var; - void *_tmp_127_var_1; + void *_tmp_128_var; + void *_tmp_128_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_127_var = _tmp_127_rule(p)) // param_no_default | ',' + (_tmp_128_var = _tmp_128_rule(p)) // param_no_default | ',' && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_127_var_1 = _tmp_127_rule(p)) // param_no_default | ',' + (_tmp_128_var_1 = _tmp_128_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); @@ -22663,7 +22719,7 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_128_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_129_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); @@ -22800,10 +22856,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); asdl_seq * _loop0_75_var; - void *_tmp_129_var; + void *_tmp_130_var; Token * a; if ( - (_tmp_129_var = _tmp_129_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_130_var = _tmp_130_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && @@ -22863,7 +22919,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_131_var; + asdl_seq * _gather_132_var; asdl_seq * _loop0_71_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22874,7 +22930,7 @@ invalid_lambda_parameters_rule(Parser *p) && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_131_var = _gather_131_rule(p)) // ','.lambda_param+ + (_gather_132_var = _gather_132_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22905,16 +22961,16 @@ invalid_lambda_parameters_rule(Parser *p) asdl_seq * _loop0_75_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_132_var; + void *_tmp_133_var; Token * a; if ( - (_opt_var = _tmp_129_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_130_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_132_var = _tmp_132_rule(p)) // ',' | lambda_param_no_default + (_tmp_133_var = _tmp_133_rule(p)) // ',' | lambda_param_no_default && (_loop0_75_var_1 = _loop0_75_rule(p)) // lambda_param_maybe_default* && @@ -23057,11 +23113,11 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_133_var; + void *_tmp_134_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_133_var = _tmp_133_rule(p)) // ':' | ',' (':' | '**') + (_tmp_134_var = _tmp_134_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); @@ -23115,19 +23171,19 @@ invalid_lambda_star_etc_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; asdl_seq * _loop0_75_var; - void *_tmp_134_var; - void *_tmp_134_var_1; + void *_tmp_135_var; + void *_tmp_135_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_134_var = _tmp_134_rule(p)) // lambda_param_no_default | ',' + (_tmp_135_var = _tmp_135_rule(p)) // lambda_param_no_default | ',' && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_134_var_1 = _tmp_134_rule(p)) // lambda_param_no_default | ',' + (_tmp_135_var_1 = _tmp_135_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); @@ -23245,7 +23301,7 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_128_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_129_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); @@ -23395,13 +23451,13 @@ invalid_for_if_clause_rule(Parser *p) Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_135_var; + void *_tmp_136_var; if ( (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 694)) // token='for' && - (_tmp_135_var = _tmp_135_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + (_tmp_136_var = _tmp_136_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 695) // token='in' ) @@ -23576,14 +23632,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_137_var; + asdl_seq * _gather_138_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 634)) // token='import' && - (_gather_137_var = _gather_137_rule(p)) // ','.dotted_name+ + (_gather_138_var = _gather_138_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 633)) // token='from' && @@ -23663,7 +23719,7 @@ invalid_dotted_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 680)) // token='as' && - _PyPegen_lookahead(0, _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_139_rule, p) && (a = expression_rule(p)) // expression ) @@ -23714,7 +23770,7 @@ invalid_import_from_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 680)) // token='as' && - _PyPegen_lookahead(0, _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_139_rule, p) && (a = expression_rule(p)) // expression ) @@ -23832,7 +23888,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_140_var; + asdl_seq * _gather_141_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -23842,7 +23898,7 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 647)) // token='with' && - (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_141_var = _gather_141_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) @@ -23866,7 +23922,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_142_var; + asdl_seq * _gather_143_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -23882,7 +23938,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_142_var = _gather_142_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_143_var = _gather_143_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23931,7 +23987,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_140_var; + asdl_seq * _gather_141_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -23942,7 +23998,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 647)) // token='with' && - (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_141_var = _gather_141_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23970,7 +24026,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_142_var; + asdl_seq * _gather_143_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -23987,7 +24043,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_142_var = _gather_142_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_143_var = _gather_143_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -24084,7 +24140,7 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, _tmp_143_rule, p) + _PyPegen_lookahead(0, _tmp_144_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); @@ -24109,7 +24165,7 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_144_var; + asdl_seq * _loop0_145_var; asdl_seq * _loop1_37_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -24121,7 +24177,7 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_144_var = _loop0_144_rule(p)) // block* + (_loop0_145_var = _loop0_145_rule(p)) // block* && (_loop1_37_var = _loop1_37_rule(p)) // except_block+ && @@ -24158,7 +24214,7 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_144_var; + asdl_seq * _loop0_145_var; asdl_seq * _loop1_38_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -24168,13 +24224,13 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_144_var = _loop0_144_rule(p)) // block* + (_loop0_145_var = _loop0_145_rule(p)) // block* && (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 677)) // token='except' && - (_opt_var = _tmp_145_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_146_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) @@ -24469,14 +24525,14 @@ invalid_except_star_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_146_var; + void *_tmp_147_var; Token * a; if ( (a = _PyPegen_expect_token(p, 677)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_146_var = _tmp_146_rule(p)) // NEWLINE | ':' + (_tmp_147_var = _tmp_147_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); @@ -25079,7 +25135,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_147_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_148_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -25811,11 +25867,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_84_var; + asdl_seq * _gather_85_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_84_var = _gather_84_rule(p)) // ','.double_starred_kvpair+ + (_gather_85_var = _gather_85_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -25823,7 +25879,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_84_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_85_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -25876,7 +25932,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -25986,7 +26042,7 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); @@ -26274,7 +26330,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26306,7 +26362,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26370,9 +26426,9 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -26409,7 +26465,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26450,7 +26506,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26497,7 +26553,7 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26716,7 +26772,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); @@ -26748,7 +26804,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); @@ -26812,9 +26868,9 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); @@ -26851,7 +26907,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26892,7 +26948,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) @@ -26939,7 +26995,7 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); @@ -26987,6 +27043,81 @@ invalid_tstring_conversion_character_rule(Parser *p) return _res; } +// invalid_string_tstring_concat: +// | ((fstring | string))+ tstring +// | tstring+ (fstring | string) +static void * +invalid_string_tstring_concat_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ((fstring | string))+ tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_81_rule(p)) // ((fstring | string))+ + && + (b = (expr_ty)tstring_rule(p)) // tstring + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+ tstring")); + } + { // tstring+ (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_82_rule(p)) // tstring+ + && + (b = (expr_ty)_tmp_154_rule(p)) // fstring | string + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if (_res == NULL && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+ (fstring | string)")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_arithmetic: sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion static void * invalid_arithmetic_rule(Parser *p) @@ -27006,14 +27137,14 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_153_var; + void *_tmp_155_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_153_var = _tmp_153_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_155_var = _tmp_155_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && (a = _PyPegen_expect_token(p, 703)) // token='not' && @@ -27058,11 +27189,11 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_154_var; + void *_tmp_156_var; Token * a; expr_ty b; if ( - (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '~' + (_tmp_156_var = _tmp_156_rule(p)) // '+' | '-' | '~' && (a = _PyPegen_expect_token(p, 703)) // token='not' && @@ -27836,12 +27967,12 @@ _loop1_12_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28174,12 +28305,12 @@ _loop0_18_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28241,12 +28372,12 @@ _loop1_19_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28593,12 +28724,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_157_var; + void *_tmp_159_var; while ( - (_tmp_157_var = _tmp_157_rule(p)) // '@' named_expression NEWLINE + (_tmp_159_var = _tmp_159_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_157_var; + _res = _tmp_159_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30626,12 +30757,12 @@ _loop1_57_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_158_var; + void *_tmp_160_var; while ( - (_tmp_158_var = _tmp_158_rule(p)) // ',' star_expression + (_tmp_160_var = _tmp_160_rule(p)) // ',' star_expression ) { - _res = _tmp_158_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30815,12 +30946,12 @@ _loop1_60_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_159_var; + void *_tmp_161_var; while ( - (_tmp_159_var = _tmp_159_rule(p)) // 'or' conjunction + (_tmp_161_var = _tmp_161_rule(p)) // 'or' conjunction ) { - _res = _tmp_159_var; + _res = _tmp_161_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30887,12 +31018,12 @@ _loop1_61_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_160_var; + void *_tmp_162_var; while ( - (_tmp_160_var = _tmp_160_rule(p)) // 'and' inversion + (_tmp_162_var = _tmp_162_rule(p)) // 'and' inversion ) { - _res = _tmp_160_var; + _res = _tmp_162_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31079,7 +31210,7 @@ _loop0_64_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression ) { _res = elem; @@ -31144,7 +31275,7 @@ _gather_65_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression && (seq = _loop0_64_rule(p)) // _loop0_64 ) @@ -32179,7 +32310,7 @@ _loop0_80_rule(Parser *p) return _seq; } -// _loop1_81: (fstring | string | tstring) +// _loop1_81: (fstring | string) static asdl_seq * _loop1_81_rule(Parser *p) { @@ -32201,18 +32332,18 @@ _loop1_81_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (fstring | string | tstring) + { // (fstring | string) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string | tstring)")); - void *_tmp_162_var; + D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_154_var; while ( - (_tmp_162_var = _tmp_162_rule(p)) // fstring | string | tstring + (_tmp_154_var = _tmp_154_rule(p)) // fstring | string ) { - _res = _tmp_162_var; + _res = _tmp_154_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32230,7 +32361,79 @@ _loop1_81_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_81[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string | tstring)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _loop1_82: tstring +static asdl_seq * +_loop1_82_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); + expr_ty tstring_var; + while ( + (tstring_var = tstring_rule(p)) // tstring + ) + { + _res = tstring_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_82[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -32251,9 +32454,9 @@ _loop1_81_rule(Parser *p) return _seq; } -// _tmp_82: star_named_expression ',' star_named_expressions? +// _tmp_83: star_named_expression ',' star_named_expressions? static void * -_tmp_82_rule(Parser *p) +_tmp_83_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32269,7 +32472,7 @@ _tmp_82_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + D(fprintf(stderr, "%*c> _tmp_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); Token * _literal; expr_ty y; void *z; @@ -32281,7 +32484,7 @@ _tmp_82_rule(Parser *p) (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { - D(fprintf(stderr, "%*c+ _tmp_82[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + D(fprintf(stderr, "%*c+ _tmp_83[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); _res = _PyPegen_seq_insert_in_front ( p , y , z ); if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -32291,7 +32494,7 @@ _tmp_82_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_82[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_83[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); } _res = NULL; @@ -32300,9 +32503,9 @@ _tmp_82_rule(Parser *p) return _res; } -// _loop0_83: ',' double_starred_kvpair +// _loop0_84: ',' double_starred_kvpair static asdl_seq * -_loop0_83_rule(Parser *p) +_loop0_84_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32327,7 +32530,7 @@ _loop0_83_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -32359,7 +32562,7 @@ _loop0_83_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_83[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_84[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32376,9 +32579,9 @@ _loop0_83_rule(Parser *p) return _seq; } -// _gather_84: double_starred_kvpair _loop0_83 +// _gather_85: double_starred_kvpair _loop0_84 static asdl_seq * -_gather_84_rule(Parser *p) +_gather_85_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32389,27 +32592,27 @@ _gather_84_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_83 + { // double_starred_kvpair _loop0_84 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_84")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_83_rule(p)) // _loop0_83 + (seq = _loop0_84_rule(p)) // _loop0_84 ) { - D(fprintf(stderr, "%*c+ _gather_84[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_84")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_84[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_84")); } _res = NULL; done: @@ -32417,9 +32620,9 @@ _gather_84_rule(Parser *p) return _res; } -// _loop1_85: for_if_clause +// _loop1_86: for_if_clause static asdl_seq * -_loop1_85_rule(Parser *p) +_loop1_86_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32444,7 +32647,7 @@ _loop1_85_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); + D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); comprehension_ty for_if_clause_var; while ( (for_if_clause_var = for_if_clause_rule(p)) // for_if_clause @@ -32467,7 +32670,7 @@ _loop1_85_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_85[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_86[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "for_if_clause")); } if (_n == 0 || p->error_indicator) { @@ -32489,9 +32692,9 @@ _loop1_85_rule(Parser *p) return _seq; } -// _loop0_86: ('if' disjunction) +// _loop0_87: ('if' disjunction) static asdl_seq * -_loop0_86_rule(Parser *p) +_loop0_87_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32516,13 +32719,13 @@ _loop0_86_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_163_var; + D(fprintf(stderr, "%*c> _loop0_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); + void *_tmp_164_var; while ( - (_tmp_163_var = _tmp_163_rule(p)) // 'if' disjunction + (_tmp_164_var = _tmp_164_rule(p)) // 'if' disjunction ) { - _res = _tmp_163_var; + _res = _tmp_164_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32539,7 +32742,7 @@ _loop0_86_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_87[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32556,9 +32759,9 @@ _loop0_86_rule(Parser *p) return _seq; } -// _tmp_87: assignment_expression | expression !':=' +// _tmp_88: assignment_expression | expression !':=' static void * -_tmp_87_rule(Parser *p) +_tmp_88_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32574,18 +32777,18 @@ _tmp_87_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_88[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_87[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_88[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -32593,7 +32796,7 @@ _tmp_87_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -32601,12 +32804,12 @@ _tmp_87_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_88[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_87[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_88[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -32615,9 +32818,9 @@ _tmp_87_rule(Parser *p) return _res; } -// _loop0_88: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_89: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_88_rule(Parser *p) +_loop0_89_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32642,13 +32845,13 @@ _loop0_88_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; @@ -32674,7 +32877,7 @@ _loop0_88_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_88[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_89[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32691,10 +32894,10 @@ _loop0_88_rule(Parser *p) return _seq; } -// _gather_89: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88 +// _gather_90: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89 static asdl_seq * -_gather_89_rule(Parser *p) +_gather_90_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32705,27 +32908,27 @@ _gather_89_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c> _gather_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_88_rule(p)) // _loop0_88 + (seq = _loop0_89_rule(p)) // _loop0_89 ) { - D(fprintf(stderr, "%*c+ _gather_89[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c+ _gather_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_89[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c%s _gather_90[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); } _res = NULL; done: @@ -32733,9 +32936,9 @@ _gather_89_rule(Parser *p) return _res; } -// _tmp_90: ',' kwargs +// _tmp_91: ',' kwargs static void * -_tmp_90_rule(Parser *p) +_tmp_91_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32751,7 +32954,7 @@ _tmp_90_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); Token * _literal; asdl_seq* k; if ( @@ -32760,7 +32963,7 @@ _tmp_90_rule(Parser *p) (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_91[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); _res = k; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -32770,7 +32973,7 @@ _tmp_90_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_90[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_91[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; @@ -32779,9 +32982,9 @@ _tmp_90_rule(Parser *p) return _res; } -// _loop0_91: ',' kwarg_or_starred +// _loop0_92: ',' kwarg_or_starred static asdl_seq * -_loop0_91_rule(Parser *p) +_loop0_92_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32806,7 +33009,7 @@ _loop0_91_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32838,7 +33041,7 @@ _loop0_91_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_91[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_92[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32855,9 +33058,9 @@ _loop0_91_rule(Parser *p) return _seq; } -// _gather_92: kwarg_or_starred _loop0_91 +// _gather_93: kwarg_or_starred _loop0_92 static asdl_seq * -_gather_92_rule(Parser *p) +_gather_93_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32868,27 +33071,27 @@ _gather_92_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_91 + { // kwarg_or_starred _loop0_92 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c> _gather_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_92")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_91_rule(p)) // _loop0_91 + (seq = _loop0_92_rule(p)) // _loop0_92 ) { - D(fprintf(stderr, "%*c+ _gather_92[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c+ _gather_93[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_92")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_92[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c%s _gather_93[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_92")); } _res = NULL; done: @@ -32896,9 +33099,9 @@ _gather_92_rule(Parser *p) return _res; } -// _loop0_93: ',' kwarg_or_double_starred +// _loop0_94: ',' kwarg_or_double_starred static asdl_seq * -_loop0_93_rule(Parser *p) +_loop0_94_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32923,7 +33126,7 @@ _loop0_93_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32955,7 +33158,7 @@ _loop0_93_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_93[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_94[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32972,9 +33175,9 @@ _loop0_93_rule(Parser *p) return _seq; } -// _gather_94: kwarg_or_double_starred _loop0_93 +// _gather_95: kwarg_or_double_starred _loop0_94 static asdl_seq * -_gather_94_rule(Parser *p) +_gather_95_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32985,27 +33188,27 @@ _gather_94_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_93 + { // kwarg_or_double_starred _loop0_94 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c> _gather_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_94")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_93_rule(p)) // _loop0_93 + (seq = _loop0_94_rule(p)) // _loop0_94 ) { - D(fprintf(stderr, "%*c+ _gather_94[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c+ _gather_95[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_94")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_94[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c%s _gather_95[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_94")); } _res = NULL; done: @@ -33013,9 +33216,9 @@ _gather_94_rule(Parser *p) return _res; } -// _loop0_95: (',' star_target) +// _loop0_96: (',' star_target) static asdl_seq * -_loop0_95_rule(Parser *p) +_loop0_96_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33040,13 +33243,13 @@ _loop0_95_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + D(fprintf(stderr, "%*c> _loop0_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33063,7 +33266,7 @@ _loop0_95_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_95[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_96[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33080,9 +33283,9 @@ _loop0_95_rule(Parser *p) return _seq; } -// _loop0_96: ',' star_target +// _loop0_97: ',' star_target static asdl_seq * -_loop0_96_rule(Parser *p) +_loop0_97_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33107,7 +33310,7 @@ _loop0_96_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _loop0_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty elem; while ( @@ -33139,7 +33342,7 @@ _loop0_96_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_96[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_97[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33156,9 +33359,9 @@ _loop0_96_rule(Parser *p) return _seq; } -// _gather_97: star_target _loop0_96 +// _gather_98: star_target _loop0_97 static asdl_seq * -_gather_97_rule(Parser *p) +_gather_98_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33169,27 +33372,27 @@ _gather_97_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_target _loop0_96 + { // star_target _loop0_97 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c> _gather_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_97")); expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) // star_target && - (seq = _loop0_96_rule(p)) // _loop0_96 + (seq = _loop0_97_rule(p)) // _loop0_97 ) { - D(fprintf(stderr, "%*c+ _gather_97[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c+ _gather_98[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_97")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_97[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c%s _gather_98[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_97")); } _res = NULL; done: @@ -33197,9 +33400,9 @@ _gather_97_rule(Parser *p) return _res; } -// _loop1_98: (',' star_target) +// _loop1_99: (',' star_target) static asdl_seq * -_loop1_98_rule(Parser *p) +_loop1_99_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33224,13 +33427,13 @@ _loop1_98_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + D(fprintf(stderr, "%*c> _loop1_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33247,7 +33450,7 @@ _loop1_98_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_98[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_99[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { @@ -33269,9 +33472,9 @@ _loop1_98_rule(Parser *p) return _seq; } -// _tmp_99: !'*' star_target +// _tmp_100: !'*' star_target static void * -_tmp_99_rule(Parser *p) +_tmp_100_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33287,7 +33490,7 @@ _tmp_99_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c> _tmp_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); expr_ty star_target_var; if ( _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' @@ -33295,12 +33498,12 @@ _tmp_99_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_99[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c+ _tmp_100[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); _res = star_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_99[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_100[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } _res = NULL; @@ -33309,9 +33512,9 @@ _tmp_99_rule(Parser *p) return _res; } -// _loop0_100: ',' del_target +// _loop0_101: ',' del_target static asdl_seq * -_loop0_100_rule(Parser *p) +_loop0_101_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33336,7 +33539,7 @@ _loop0_100_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + D(fprintf(stderr, "%*c> _loop0_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); Token * _literal; expr_ty elem; while ( @@ -33368,7 +33571,7 @@ _loop0_100_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_100[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_101[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33385,9 +33588,9 @@ _loop0_100_rule(Parser *p) return _seq; } -// _gather_101: del_target _loop0_100 +// _gather_102: del_target _loop0_101 static asdl_seq * -_gather_101_rule(Parser *p) +_gather_102_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33398,27 +33601,27 @@ _gather_101_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // del_target _loop0_100 + { // del_target _loop0_101 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c> _gather_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_101")); expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) // del_target && - (seq = _loop0_100_rule(p)) // _loop0_100 + (seq = _loop0_101_rule(p)) // _loop0_101 ) { - D(fprintf(stderr, "%*c+ _gather_101[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c+ _gather_102[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_101")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_101[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c%s _gather_102[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_101")); } _res = NULL; done: @@ -33426,9 +33629,9 @@ _gather_101_rule(Parser *p) return _res; } -// _loop0_102: ',' expression +// _loop0_103: ',' expression static asdl_seq * -_loop0_102_rule(Parser *p) +_loop0_103_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33453,7 +33656,7 @@ _loop0_102_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33485,7 +33688,7 @@ _loop0_102_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_102[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_103[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33502,9 +33705,9 @@ _loop0_102_rule(Parser *p) return _seq; } -// _gather_103: expression _loop0_102 +// _gather_104: expression _loop0_103 static asdl_seq * -_gather_103_rule(Parser *p) +_gather_104_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33515,27 +33718,27 @@ _gather_103_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_102 + { // expression _loop0_103 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c> _gather_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_103")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_102_rule(p)) // _loop0_102 + (seq = _loop0_103_rule(p)) // _loop0_103 ) { - D(fprintf(stderr, "%*c+ _gather_103[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c+ _gather_104[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_103")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_103[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c%s _gather_104[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_103")); } _res = NULL; done: @@ -33543,9 +33746,9 @@ _gather_103_rule(Parser *p) return _res; } -// _tmp_104: NEWLINE INDENT +// _tmp_105: NEWLINE INDENT static void * -_tmp_104_rule(Parser *p) +_tmp_105_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33561,7 +33764,7 @@ _tmp_104_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); Token * indent_var; Token * newline_var; if ( @@ -33570,12 +33773,12 @@ _tmp_104_rule(Parser *p) (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - D(fprintf(stderr, "%*c+ _tmp_104[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); _res = _PyPegen_dummy_name(p, newline_var, indent_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_104[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } _res = NULL; @@ -33584,11 +33787,11 @@ _tmp_104_rule(Parser *p) return _res; } -// _tmp_105: +// _tmp_106: // | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) // | kwargs static void * -_tmp_105_rule(Parser *p) +_tmp_106_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33604,18 +33807,18 @@ _tmp_105_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_166_var; + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_167_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_167_var = _tmp_167_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_166_var; + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_167_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } { // kwargs @@ -33623,18 +33826,18 @@ _tmp_105_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); asdl_seq* kwargs_var; if ( (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); _res = kwargs_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } _res = NULL; @@ -33643,9 +33846,9 @@ _tmp_105_rule(Parser *p) return _res; } -// _loop0_106: ',' (starred_expression !'=') +// _loop0_107: ',' (starred_expression !'=') static asdl_seq * -_loop0_106_rule(Parser *p) +_loop0_107_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33670,13 +33873,13 @@ _loop0_106_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + D(fprintf(stderr, "%*c> _loop0_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' ) { _res = elem; @@ -33702,7 +33905,7 @@ _loop0_106_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_106[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_107[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33719,9 +33922,9 @@ _loop0_106_rule(Parser *p) return _seq; } -// _gather_107: (starred_expression !'=') _loop0_106 +// _gather_108: (starred_expression !'=') _loop0_107 static asdl_seq * -_gather_107_rule(Parser *p) +_gather_108_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33732,27 +33935,27 @@ _gather_107_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression !'=') _loop0_106 + { // (starred_expression !'=') _loop0_107 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c> _gather_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_107")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' && - (seq = _loop0_106_rule(p)) // _loop0_106 + (seq = _loop0_107_rule(p)) // _loop0_107 ) { - D(fprintf(stderr, "%*c+ _gather_107[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c+ _gather_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_107")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_107[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c%s _gather_108[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_107")); } _res = NULL; done: @@ -33760,9 +33963,9 @@ _gather_107_rule(Parser *p) return _res; } -// _tmp_108: args | expression for_if_clauses +// _tmp_109: args | expression for_if_clauses static void * -_tmp_108_rule(Parser *p) +_tmp_109_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33778,18 +33981,18 @@ _tmp_108_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -33797,7 +34000,7 @@ _tmp_108_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -33806,12 +34009,12 @@ _tmp_108_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -33820,9 +34023,9 @@ _tmp_108_rule(Parser *p) return _res; } -// _tmp_109: args ',' +// _tmp_110: args ',' static void * -_tmp_109_rule(Parser *p) +_tmp_110_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33838,7 +34041,7 @@ _tmp_109_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -33847,12 +34050,12 @@ _tmp_109_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -33861,9 +34064,9 @@ _tmp_109_rule(Parser *p) return _res; } -// _tmp_110: ',' | ')' +// _tmp_111: ',' | ')' static void * -_tmp_110_rule(Parser *p) +_tmp_111_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33879,18 +34082,18 @@ _tmp_110_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -33898,18 +34101,18 @@ _tmp_110_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -33918,9 +34121,9 @@ _tmp_110_rule(Parser *p) return _res; } -// _tmp_111: 'True' | 'False' | 'None' +// _tmp_112: 'True' | 'False' | 'None' static void * -_tmp_111_rule(Parser *p) +_tmp_112_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33936,18 +34139,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 622)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -33955,18 +34158,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 624)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -33974,18 +34177,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 623)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -33994,9 +34197,9 @@ _tmp_111_rule(Parser *p) return _res; } -// _tmp_112: NAME '=' +// _tmp_113: NAME '=' static void * -_tmp_112_rule(Parser *p) +_tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34012,7 +34215,7 @@ _tmp_112_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -34021,12 +34224,12 @@ _tmp_112_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_113[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_113[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -34035,9 +34238,9 @@ _tmp_112_rule(Parser *p) return _res; } -// _loop1_113: (!STRING expression_without_invalid) +// _loop1_114: (!STRING expression_without_invalid) static asdl_seq * -_loop1_113_rule(Parser *p) +_loop1_114_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34062,13 +34265,13 @@ _loop1_113_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(!STRING expression_without_invalid)")); - void *_tmp_168_var; + D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(!STRING expression_without_invalid)")); + void *_tmp_169_var; while ( - (_tmp_168_var = _tmp_168_rule(p)) // !STRING expression_without_invalid + (_tmp_169_var = _tmp_169_rule(p)) // !STRING expression_without_invalid ) { - _res = _tmp_168_var; + _res = _tmp_169_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34085,7 +34288,7 @@ _loop1_113_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_113[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_114[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(!STRING expression_without_invalid)")); } if (_n == 0 || p->error_indicator) { @@ -34107,9 +34310,9 @@ _loop1_113_rule(Parser *p) return _seq; } -// _tmp_114: NAME STRING | SOFT_KEYWORD +// _tmp_115: NAME STRING | SOFT_KEYWORD static void * -_tmp_114_rule(Parser *p) +_tmp_115_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34125,7 +34328,7 @@ _tmp_114_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -34134,12 +34337,12 @@ _tmp_114_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -34147,18 +34350,18 @@ _tmp_114_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -34167,9 +34370,9 @@ _tmp_114_rule(Parser *p) return _res; } -// _tmp_115: 'else' | ':' +// _tmp_116: 'else' | ':' static void * -_tmp_115_rule(Parser *p) +_tmp_116_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34185,18 +34388,18 @@ _tmp_115_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 686)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -34204,18 +34407,18 @@ _tmp_115_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -34224,9 +34427,9 @@ _tmp_115_rule(Parser *p) return _res; } -// _tmp_116: pass_stmt | break_stmt | continue_stmt +// _tmp_117: pass_stmt | break_stmt | continue_stmt static void * -_tmp_116_rule(Parser *p) +_tmp_117_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34242,18 +34445,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pass_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pass_stmt")); stmt_ty pass_stmt_var; if ( (pass_stmt_var = pass_stmt_rule(p)) // pass_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pass_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pass_stmt")); _res = pass_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pass_stmt")); } { // break_stmt @@ -34261,18 +34464,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "break_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "break_stmt")); stmt_ty break_stmt_var; if ( (break_stmt_var = break_stmt_rule(p)) // break_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "break_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "break_stmt")); _res = break_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "break_stmt")); } { // continue_stmt @@ -34280,18 +34483,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "continue_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "continue_stmt")); stmt_ty continue_stmt_var; if ( (continue_stmt_var = continue_stmt_rule(p)) // continue_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "continue_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "continue_stmt")); _res = continue_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "continue_stmt")); } _res = NULL; @@ -34300,9 +34503,9 @@ _tmp_116_rule(Parser *p) return _res; } -// _tmp_117: '=' | ':=' +// _tmp_118: '=' | ':=' static void * -_tmp_117_rule(Parser *p) +_tmp_118_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34318,18 +34521,18 @@ _tmp_117_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34337,18 +34540,18 @@ _tmp_117_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34357,9 +34560,9 @@ _tmp_117_rule(Parser *p) return _res; } -// _tmp_118: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_119: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_118_rule(Parser *p) +_tmp_119_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34375,18 +34578,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34394,18 +34597,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34413,18 +34616,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34432,18 +34635,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 622)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34451,18 +34654,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 623)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34470,18 +34673,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 624)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34490,9 +34693,9 @@ _tmp_118_rule(Parser *p) return _res; } -// _loop0_119: star_named_expressions +// _loop0_120: star_named_expressions static asdl_seq * -_loop0_119_rule(Parser *p) +_loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34517,7 +34720,7 @@ _loop0_119_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -34540,7 +34743,7 @@ _loop0_119_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_119[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34557,9 +34760,9 @@ _loop0_119_rule(Parser *p) return _seq; } -// _loop0_120: (star_targets '=') +// _loop0_121: (star_targets '=') static asdl_seq * -_loop0_120_rule(Parser *p) +_loop0_121_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34584,13 +34787,13 @@ _loop0_120_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34607,7 +34810,7 @@ _loop0_120_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34624,9 +34827,9 @@ _loop0_120_rule(Parser *p) return _seq; } -// _tmp_121: '[' | '(' | '{' +// _tmp_122: '[' | '(' | '{' static void * -_tmp_121_rule(Parser *p) +_tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34642,18 +34845,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -34661,18 +34864,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -34680,18 +34883,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -34700,9 +34903,9 @@ _tmp_121_rule(Parser *p) return _res; } -// _tmp_122: '[' | '{' +// _tmp_123: '[' | '{' static void * -_tmp_122_rule(Parser *p) +_tmp_123_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34718,18 +34921,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -34737,18 +34940,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -34757,9 +34960,9 @@ _tmp_122_rule(Parser *p) return _res; } -// _tmp_123: slash_no_default | slash_with_default +// _tmp_124: slash_no_default | slash_with_default static void * -_tmp_123_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34775,18 +34978,18 @@ _tmp_123_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -34794,18 +34997,18 @@ _tmp_123_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -34814,9 +35017,9 @@ _tmp_123_rule(Parser *p) return _res; } -// _tmp_124: ',' | param_no_default +// _tmp_125: ',' | param_no_default static void * -_tmp_124_rule(Parser *p) +_tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34832,18 +35035,18 @@ _tmp_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -34851,18 +35054,18 @@ _tmp_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -34871,9 +35074,9 @@ _tmp_124_rule(Parser *p) return _res; } -// _tmp_125: ')' | ',' +// _tmp_126: ')' | ',' static void * -_tmp_125_rule(Parser *p) +_tmp_126_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34889,18 +35092,18 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -34908,18 +35111,18 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -34928,9 +35131,9 @@ _tmp_125_rule(Parser *p) return _res; } -// _tmp_126: ')' | ',' (')' | '**') +// _tmp_127: ')' | ',' (')' | '**') static void * -_tmp_126_rule(Parser *p) +_tmp_127_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34946,18 +35149,18 @@ _tmp_126_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -34965,21 +35168,21 @@ _tmp_126_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_169_var; + void *_tmp_170_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_169_var = _tmp_169_rule(p)) // ')' | '**' + (_tmp_170_var = _tmp_170_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_169_var); + D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -34988,9 +35191,9 @@ _tmp_126_rule(Parser *p) return _res; } -// _tmp_127: param_no_default | ',' +// _tmp_128: param_no_default | ',' static void * -_tmp_127_rule(Parser *p) +_tmp_128_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35006,18 +35209,18 @@ _tmp_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -35025,18 +35228,18 @@ _tmp_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35045,9 +35248,9 @@ _tmp_127_rule(Parser *p) return _res; } -// _tmp_128: '*' | '**' | '/' +// _tmp_129: '*' | '**' | '/' static void * -_tmp_128_rule(Parser *p) +_tmp_129_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35063,18 +35266,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -35082,18 +35285,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -35101,18 +35304,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -35121,9 +35324,9 @@ _tmp_128_rule(Parser *p) return _res; } -// _tmp_129: lambda_slash_no_default | lambda_slash_with_default +// _tmp_130: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_129_rule(Parser *p) +_tmp_130_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35139,18 +35342,18 @@ _tmp_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -35158,18 +35361,18 @@ _tmp_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -35178,9 +35381,9 @@ _tmp_129_rule(Parser *p) return _res; } -// _loop0_130: ',' lambda_param +// _loop0_131: ',' lambda_param static asdl_seq * -_loop0_130_rule(Parser *p) +_loop0_131_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35205,7 +35408,7 @@ _loop0_130_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -35237,7 +35440,7 @@ _loop0_130_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_131[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35254,9 +35457,9 @@ _loop0_130_rule(Parser *p) return _seq; } -// _gather_131: lambda_param _loop0_130 +// _gather_132: lambda_param _loop0_131 static asdl_seq * -_gather_131_rule(Parser *p) +_gather_132_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35267,27 +35470,27 @@ _gather_131_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_130 + { // lambda_param _loop0_131 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c> _gather_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_131")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_130_rule(p)) // _loop0_130 + (seq = _loop0_131_rule(p)) // _loop0_131 ) { - D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c+ _gather_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_131")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c%s _gather_132[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_131")); } _res = NULL; done: @@ -35295,9 +35498,9 @@ _gather_131_rule(Parser *p) return _res; } -// _tmp_132: ',' | lambda_param_no_default +// _tmp_133: ',' | lambda_param_no_default static void * -_tmp_132_rule(Parser *p) +_tmp_133_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35313,18 +35516,18 @@ _tmp_132_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_132[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -35332,18 +35535,18 @@ _tmp_132_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_132[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -35352,9 +35555,9 @@ _tmp_132_rule(Parser *p) return _res; } -// _tmp_133: ':' | ',' (':' | '**') +// _tmp_134: ':' | ',' (':' | '**') static void * -_tmp_133_rule(Parser *p) +_tmp_134_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35370,18 +35573,18 @@ _tmp_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -35389,21 +35592,21 @@ _tmp_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_170_var; + void *_tmp_171_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_170_var = _tmp_170_rule(p)) // ':' | '**' + (_tmp_171_var = _tmp_171_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); + D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_171_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -35412,9 +35615,9 @@ _tmp_133_rule(Parser *p) return _res; } -// _tmp_134: lambda_param_no_default | ',' +// _tmp_135: lambda_param_no_default | ',' static void * -_tmp_134_rule(Parser *p) +_tmp_135_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35430,18 +35633,18 @@ _tmp_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -35449,18 +35652,18 @@ _tmp_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35469,9 +35672,9 @@ _tmp_134_rule(Parser *p) return _res; } -// _tmp_135: bitwise_or ((',' bitwise_or))* ','? +// _tmp_136: bitwise_or ((',' bitwise_or))* ','? static void * -_tmp_135_rule(Parser *p) +_tmp_136_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35487,25 +35690,25 @@ _tmp_135_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_171_var; + D(fprintf(stderr, "%*c> _tmp_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_172_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_171_var = _loop0_171_rule(p)) // ((',' bitwise_or))* + (_loop0_172_var = _loop0_172_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { - D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_171_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_136[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_172_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_136[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); } _res = NULL; @@ -35514,9 +35717,9 @@ _tmp_135_rule(Parser *p) return _res; } -// _loop0_136: ',' dotted_name +// _loop0_137: ',' dotted_name static asdl_seq * -_loop0_136_rule(Parser *p) +_loop0_137_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35541,7 +35744,7 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -35573,7 +35776,7 @@ _loop0_136_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_137[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35590,9 +35793,9 @@ _loop0_136_rule(Parser *p) return _seq; } -// _gather_137: dotted_name _loop0_136 +// _gather_138: dotted_name _loop0_137 static asdl_seq * -_gather_137_rule(Parser *p) +_gather_138_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35603,27 +35806,27 @@ _gather_137_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_136 + { // dotted_name _loop0_137 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c> _gather_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_137")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_136_rule(p)) // _loop0_136 + (seq = _loop0_137_rule(p)) // _loop0_137 ) { - D(fprintf(stderr, "%*c+ _gather_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c+ _gather_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_137")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_137[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c%s _gather_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_137")); } _res = NULL; done: @@ -35631,9 +35834,9 @@ _gather_137_rule(Parser *p) return _res; } -// _tmp_138: NAME (',' | ')' | NEWLINE) +// _tmp_139: NAME (',' | ')' | NEWLINE) static void * -_tmp_138_rule(Parser *p) +_tmp_139_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35649,21 +35852,21 @@ _tmp_138_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - void *_tmp_172_var; + D(fprintf(stderr, "%*c> _tmp_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); + void *_tmp_173_var; expr_ty name_var; if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_tmp_172_var = _tmp_172_rule(p)) // ',' | ')' | NEWLINE + (_tmp_173_var = _tmp_173_rule(p)) // ',' | ')' | NEWLINE ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - _res = _PyPegen_dummy_name(p, name_var, _tmp_172_var); + D(fprintf(stderr, "%*c+ _tmp_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); + _res = _PyPegen_dummy_name(p, name_var, _tmp_173_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_139[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); } _res = NULL; @@ -35672,9 +35875,9 @@ _tmp_138_rule(Parser *p) return _res; } -// _loop0_139: ',' (expression ['as' star_target]) +// _loop0_140: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_139_rule(Parser *p) +_loop0_140_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35699,13 +35902,13 @@ _loop0_139_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] ) { _res = elem; @@ -35731,7 +35934,7 @@ _loop0_139_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_140[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35748,9 +35951,9 @@ _loop0_139_rule(Parser *p) return _seq; } -// _gather_140: (expression ['as' star_target]) _loop0_139 +// _gather_141: (expression ['as' star_target]) _loop0_140 static asdl_seq * -_gather_140_rule(Parser *p) +_gather_141_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35761,27 +35964,27 @@ _gather_140_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_139 + { // (expression ['as' star_target]) _loop0_140 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c> _gather_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] && - (seq = _loop0_139_rule(p)) // _loop0_139 + (seq = _loop0_140_rule(p)) // _loop0_140 ) { - D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c+ _gather_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c%s _gather_141[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); } _res = NULL; done: @@ -35789,9 +35992,9 @@ _gather_140_rule(Parser *p) return _res; } -// _loop0_141: ',' (expressions ['as' star_target]) +// _loop0_142: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_141_rule(Parser *p) +_loop0_142_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35816,13 +36019,13 @@ _loop0_141_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] ) { _res = elem; @@ -35848,7 +36051,7 @@ _loop0_141_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_142[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35865,9 +36068,9 @@ _loop0_141_rule(Parser *p) return _seq; } -// _gather_142: (expressions ['as' star_target]) _loop0_141 +// _gather_143: (expressions ['as' star_target]) _loop0_142 static asdl_seq * -_gather_142_rule(Parser *p) +_gather_143_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35878,27 +36081,27 @@ _gather_142_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_141 + { // (expressions ['as' star_target]) _loop0_142 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c> _gather_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_141_rule(p)) // _loop0_141 + (seq = _loop0_142_rule(p)) // _loop0_142 ) { - D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c+ _gather_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c%s _gather_143[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); } _res = NULL; done: @@ -35906,9 +36109,9 @@ _gather_142_rule(Parser *p) return _res; } -// _tmp_143: 'except' | 'finally' +// _tmp_144: 'except' | 'finally' static void * -_tmp_143_rule(Parser *p) +_tmp_144_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35924,18 +36127,18 @@ _tmp_143_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -35943,18 +36146,18 @@ _tmp_143_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -35963,9 +36166,9 @@ _tmp_143_rule(Parser *p) return _res; } -// _loop0_144: block +// _loop0_145: block static asdl_seq * -_loop0_144_rule(Parser *p) +_loop0_145_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35990,7 +36193,7 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -36013,7 +36216,7 @@ _loop0_144_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_144[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36030,9 +36233,9 @@ _loop0_144_rule(Parser *p) return _seq; } -// _tmp_145: expression ['as' NAME] +// _tmp_146: expression ['as' NAME] static void * -_tmp_145_rule(Parser *p) +_tmp_146_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36048,7 +36251,7 @@ _tmp_145_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; @@ -36058,12 +36261,12 @@ _tmp_145_rule(Parser *p) (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_145[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -36072,9 +36275,9 @@ _tmp_145_rule(Parser *p) return _res; } -// _tmp_146: NEWLINE | ':' +// _tmp_147: NEWLINE | ':' static void * -_tmp_146_rule(Parser *p) +_tmp_147_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36090,18 +36293,18 @@ _tmp_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -36109,18 +36312,18 @@ _tmp_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -36129,9 +36332,9 @@ _tmp_146_rule(Parser *p) return _res; } -// _tmp_147: positional_patterns ',' +// _tmp_148: positional_patterns ',' static void * -_tmp_147_rule(Parser *p) +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36147,7 +36350,7 @@ _tmp_147_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -36156,12 +36359,12 @@ _tmp_147_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -36170,9 +36373,9 @@ _tmp_147_rule(Parser *p) return _res; } -// _tmp_148: '}' | ',' +// _tmp_149: '}' | ',' static void * -_tmp_148_rule(Parser *p) +_tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36188,18 +36391,18 @@ _tmp_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -36207,18 +36410,18 @@ _tmp_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36227,9 +36430,9 @@ _tmp_148_rule(Parser *p) return _res; } -// _tmp_149: '=' | '!' | ':' | '}' +// _tmp_150: '=' | '!' | ':' | '}' static void * -_tmp_149_rule(Parser *p) +_tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36245,18 +36448,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -36264,18 +36467,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -36283,18 +36486,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36302,18 +36505,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36322,9 +36525,9 @@ _tmp_149_rule(Parser *p) return _res; } -// _tmp_150: '!' | ':' | '}' +// _tmp_151: '!' | ':' | '}' static void * -_tmp_150_rule(Parser *p) +_tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36340,18 +36543,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -36359,18 +36562,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36378,18 +36581,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36398,9 +36601,9 @@ _tmp_150_rule(Parser *p) return _res; } -// _tmp_151: '!' NAME +// _tmp_152: '!' NAME static void * -_tmp_151_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36416,7 +36619,7 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -36425,12 +36628,12 @@ _tmp_151_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -36439,9 +36642,9 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: ':' | '}' +// _tmp_153: ':' | '}' static void * -_tmp_152_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36457,18 +36660,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36476,18 +36679,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36496,9 +36699,66 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_154: fstring | string static void * -_tmp_153_rule(Parser *p) +_tmp_154_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // fstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + expr_ty fstring_var; + if ( + (fstring_var = fstring_rule(p)) // fstring + ) + { + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + _res = fstring_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); + } + { // string + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + expr_ty string_var; + if ( + (string_var = string_rule(p)) // string + ) + { + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + _res = string_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_155: '+' | '-' | '*' | '/' | '%' | '//' | '@' +static void * +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36514,18 +36774,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36533,18 +36793,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -36552,18 +36812,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -36571,18 +36831,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -36590,18 +36850,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -36609,18 +36869,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -36628,18 +36888,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -36648,9 +36908,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: '+' | '-' | '~' +// _tmp_156: '+' | '-' | '~' static void * -_tmp_154_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36666,18 +36926,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36685,18 +36945,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -36704,18 +36964,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -36724,9 +36984,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: star_targets '=' +// _tmp_157: star_targets '=' static void * -_tmp_155_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36742,7 +37002,7 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -36751,7 +37011,7 @@ _tmp_155_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36761,7 +37021,7 @@ _tmp_155_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -36770,9 +37030,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: '.' | '...' +// _tmp_158: '.' | '...' static void * -_tmp_156_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36788,18 +37048,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -36807,18 +37067,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -36827,9 +37087,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: '@' named_expression NEWLINE +// _tmp_159: '@' named_expression NEWLINE static void * -_tmp_157_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36845,7 +37105,7 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -36857,7 +37117,7 @@ _tmp_157_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36867,7 +37127,7 @@ _tmp_157_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -36876,9 +37136,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: ',' star_expression +// _tmp_160: ',' star_expression static void * -_tmp_158_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36894,7 +37154,7 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -36903,7 +37163,7 @@ _tmp_158_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36913,7 +37173,7 @@ _tmp_158_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -36922,9 +37182,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: 'or' conjunction +// _tmp_161: 'or' conjunction static void * -_tmp_159_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36940,7 +37200,7 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -36949,7 +37209,7 @@ _tmp_159_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -36959,7 +37219,7 @@ _tmp_159_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -36968,9 +37228,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: 'and' inversion +// _tmp_162: 'and' inversion static void * -_tmp_160_rule(Parser *p) +_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36986,7 +37246,7 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -36995,7 +37255,7 @@ _tmp_160_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37005,7 +37265,7 @@ _tmp_160_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -37014,9 +37274,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _tmp_161: slice | starred_expression +// _tmp_163: slice | starred_expression static void * -_tmp_161_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37032,18 +37292,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -37051,18 +37311,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -37071,85 +37331,9 @@ _tmp_161_rule(Parser *p) return _res; } -// _tmp_162: fstring | string | tstring +// _tmp_164: 'if' disjunction static void * -_tmp_162_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // fstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); - expr_ty fstring_var; - if ( - (fstring_var = fstring_rule(p)) // fstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); - _res = fstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); - } - { // string - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); - expr_ty string_var; - if ( - (string_var = string_rule(p)) // string - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); - _res = string_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); - } - { // tstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); - expr_ty tstring_var; - if ( - (tstring_var = tstring_rule(p)) // tstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring")); - _res = tstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_163: 'if' disjunction -static void * -_tmp_163_rule(Parser *p) +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37165,7 +37349,7 @@ _tmp_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -37174,7 +37358,7 @@ _tmp_163_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37184,7 +37368,7 @@ _tmp_163_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -37193,9 +37377,9 @@ _tmp_163_rule(Parser *p) return _res; } -// _tmp_164: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_165: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_164_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37211,18 +37395,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -37230,20 +37414,20 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_87_var; + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_88_var; if ( - (_tmp_87_var = _tmp_87_rule(p)) // assignment_expression | expression !':=' + (_tmp_88_var = _tmp_88_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_87_var; + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_88_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -37252,9 +37436,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: ',' star_target +// _tmp_166: ',' star_target static void * -_tmp_165_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37270,7 +37454,7 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -37279,7 +37463,7 @@ _tmp_165_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; if (_res == NULL && PyErr_Occurred()) { p->error_indicator = 1; @@ -37289,7 +37473,7 @@ _tmp_165_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -37298,10 +37482,10 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: +// _tmp_167: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_166_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37317,24 +37501,24 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_89_var; + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_90_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_89_var = _gather_89_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_90_var = _gather_90_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_89_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_90_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -37343,9 +37527,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: starred_expression !'=' +// _tmp_168: starred_expression !'=' static void * -_tmp_167_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37361,7 +37545,7 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -37369,12 +37553,12 @@ _tmp_167_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -37383,9 +37567,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: !STRING expression_without_invalid +// _tmp_169: !STRING expression_without_invalid static void * -_tmp_168_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37401,7 +37585,7 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); expr_ty expression_without_invalid_var; if ( _PyPegen_lookahead(0, _PyPegen_string_token, p) @@ -37409,12 +37593,12 @@ _tmp_168_rule(Parser *p) (expression_without_invalid_var = expression_without_invalid_rule(p)) // expression_without_invalid ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); _res = expression_without_invalid_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!STRING expression_without_invalid")); } _res = NULL; @@ -37423,9 +37607,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _tmp_169: ')' | '**' +// _tmp_170: ')' | '**' static void * -_tmp_169_rule(Parser *p) +_tmp_170_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37441,18 +37625,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -37460,18 +37644,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -37480,9 +37664,9 @@ _tmp_169_rule(Parser *p) return _res; } -// _tmp_170: ':' | '**' +// _tmp_171: ':' | '**' static void * -_tmp_170_rule(Parser *p) +_tmp_171_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37498,18 +37682,18 @@ _tmp_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -37517,18 +37701,18 @@ _tmp_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -37537,9 +37721,9 @@ _tmp_170_rule(Parser *p) return _res; } -// _loop0_171: (',' bitwise_or) +// _loop0_172: (',' bitwise_or) static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_172_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37564,13 +37748,13 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_175_var; + D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_176_var; while ( - (_tmp_175_var = _tmp_175_rule(p)) // ',' bitwise_or + (_tmp_176_var = _tmp_176_rule(p)) // ',' bitwise_or ) { - _res = _tmp_175_var; + _res = _tmp_176_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -37587,7 +37771,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37604,9 +37788,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _tmp_172: ',' | ')' | NEWLINE +// _tmp_173: ',' | ')' | NEWLINE static void * -_tmp_172_rule(Parser *p) +_tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37622,18 +37806,18 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37641,18 +37825,18 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // NEWLINE @@ -37660,18 +37844,18 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -37680,9 +37864,9 @@ _tmp_172_rule(Parser *p) return _res; } -// _tmp_173: expression ['as' star_target] +// _tmp_174: expression ['as' star_target] static void * -_tmp_173_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37698,22 +37882,22 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -37722,9 +37906,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _tmp_174: expressions ['as' star_target] +// _tmp_175: expressions ['as' star_target] static void * -_tmp_174_rule(Parser *p) +_tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37740,22 +37924,22 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -37764,9 +37948,9 @@ _tmp_174_rule(Parser *p) return _res; } -// _tmp_175: ',' bitwise_or +// _tmp_176: ',' bitwise_or static void * -_tmp_175_rule(Parser *p) +_tmp_176_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37782,7 +37966,7 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -37791,12 +37975,12 @@ _tmp_175_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -37805,9 +37989,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _tmp_176: 'as' star_target +// _tmp_177: 'as' star_target static void * -_tmp_176_rule(Parser *p) +_tmp_177_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37823,7 +38007,7 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -37832,12 +38016,12 @@ _tmp_176_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; diff --git a/Parser/pegen.h b/Parser/pegen.h index 804f931871aec8..dfa2b0b4fe726c 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -351,6 +351,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok); +expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int); expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index 557c12cfda61ff..c25699978cf651 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec) } static int -_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx, - Py_ssize_t last_idx, char prefix, PyArena *arena) +append_templatestr(PyUnicodeWriter *writer, expr_ty e) { int result = -1; - - asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena); - if (!new_values) { - return result; - } - - Py_ssize_t j = 0; - for (Py_ssize_t i = first_idx; i <= last_idx; ++i) { - asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i)); - } - - PyObject *body = build_ftstring_body(new_values, false); + PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false); if (!body) { - return result; + return -1; } - if (-1 != append_char(writer, prefix) && + if (-1 != append_charp(writer, "t") && -1 != append_repr(writer, body)) { result = 0; @@ -692,72 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_ return result; } -static int -append_templatestr(PyUnicodeWriter *writer, expr_ty e) -{ - PyArena *arena = _PyArena_New(); - if (!arena) { - return -1; - } - - Py_ssize_t last_idx = 0; - Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values); - if (len == 0) { - int result = _write_values_subarray(writer, e->v.TemplateStr.values, - 0, len - 1, 't', arena); - _PyArena_Free(arena); - return result; - } - - for (Py_ssize_t i = 0; i < len; i++) { - expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i); - - // Handle implicit concat of t-strings with f-strings - if (value->kind == FormattedValue_kind) { - if (i > last_idx) { - // Create a new TemplateStr with the values between last_idx and i - // and append it to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, i - 1, 't', arena) == -1) { - goto error; - } - - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - // Append the FormattedValue to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - i, i, 'f', arena) == -1) { - goto error; - } - - if (i + 1 < len) { - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - last_idx = i + 1; - } - } - - if (last_idx < len) { - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, len - 1, 't', arena) == -1) { - goto error; - } - } - _PyArena_Free(arena); - - return 0; - -error: - _PyArena_Free(arena); - return -1; -} - static int append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) { diff --git a/Python/codegen.c b/Python/codegen.c index b3dbed7096489b..0488008cd7f16c 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -4070,16 +4070,6 @@ codegen_template_str(compiler *c, expr_ty e) } else { VISIT(c, expr, value); - Py_ssize_t j; - for (j = i + 1; j < value_count; j++) { - value = asdl_seq_GET(e->v.TemplateStr.values, j); - if (value->kind == Interpolation_kind) { - break; - } - VISIT(c, expr, value); - ADDOP_INPLACE(c, loc, Add); - } - i = j - 1; stringslen++; last_was_interpolation = 0; } From 11378e1c859d5601bb08a43ec1c8902bf2428c90 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 18:04:13 +0200 Subject: [PATCH 685/713] [3.14] gh-135621: Simplify TermInfo (GH-136916) (#136925) --- Lib/_pyrepl/terminfo.py | 108 ++++++++++++---------------------------- 1 file changed, 33 insertions(+), 75 deletions(-) diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py index 063a285bb9900c..d02ef69cce0bd8 100644 --- a/Lib/_pyrepl/terminfo.py +++ b/Lib/_pyrepl/terminfo.py @@ -71,7 +71,6 @@ "OTGV", "OTGC","meml", "memu", "box1" ) # fmt: on -_STRING_CAPABILITY_NAMES = {name: i for i, name in enumerate(_STRING_NAMES)} def _get_terminfo_dirs() -> list[Path]: @@ -322,10 +321,6 @@ class TermInfo: terminal_name: str | bytes | None fallback: bool = True - _names: list[str] = field(default_factory=list) - _booleans: list[int] = field(default_factory=list) - _numbers: list[int] = field(default_factory=list) - _strings: list[bytes | None] = field(default_factory=list) _capabilities: dict[str, bytes] = field(default_factory=dict) def __post_init__(self) -> None: @@ -362,9 +357,12 @@ def __post_init__(self) -> None: def _parse_terminfo_file(self, terminal_name: str) -> None: """Parse a terminfo file. + Populate the _capabilities dict for easy retrieval + Based on ncurses implementation in: - ncurses/tinfo/read_entry.c:_nc_read_termtype() - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() """ data = _read_terminfo_file(terminal_name) too_short = f"TermInfo file for {terminal_name!r} too short" @@ -377,53 +375,36 @@ def _parse_terminfo_file(self, terminal_name: str) -> None: ) if magic == MAGIC16: - number_format = " len(data): - raise ValueError(too_short) - names = data[offset : offset + name_size - 1].decode( - "ascii", errors="ignore" - ) + # Skip data than PyREPL doesn't need: + # - names (`|`-separated ASCII strings) + # - boolean capabilities (bytes with value 0 or 1) + # - numbers (little-endian integers, `number_size` bytes each) offset += name_size - - # Read boolean capabilities - if offset + bool_count > len(data): - raise ValueError(too_short) - booleans = list(data[offset : offset + bool_count]) offset += bool_count - - # Align to even byte boundary for numbers if offset % 2: + # Align to even byte boundary for numbers offset += 1 - - # Read numeric capabilities - numbers = [] - for i in range(num_count): - if offset + number_size > len(data): - raise ValueError(too_short) - num = struct.unpack( - number_format, data[offset : offset + number_size] - )[0] - numbers.append(num) - offset += number_size + offset += num_count * number_size + if offset > len(data): + raise ValueError(too_short) # Read string offsets - string_offsets = [] - for i in range(str_count): - if offset + 2 > len(data): - raise ValueError(too_short) - off = struct.unpack(" len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): @@ -431,53 +412,30 @@ def _parse_terminfo_file(self, terminal_name: str) -> None: string_table = data[offset : offset + str_size] # Extract strings from string table - strings: list[bytes | None] = [] - for off in string_offsets: + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): if off < 0: - strings.append(CANCELLED_STRING) + # CANCELLED_STRING; we do not store those + continue elif off < len(string_table): # Find null terminator - end = off - while end < len(string_table) and string_table[end] != 0: - end += 1 - if end <= len(string_table): - strings.append(string_table[off:end]) - else: - strings.append(ABSENT_STRING) - else: - strings.append(ABSENT_STRING) - - self._names = names.split("|") - self._booleans = booleans - self._numbers = numbers - self._strings = strings + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. - def get(self, cap: str) -> bytes | None: - """Get terminal capability string by name. + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. - Based on ncurses implementation in: - - ncurses/tinfo/lib_ti.c:tigetstr() + self._capabilities = capabilities - The ncurses version searches through compiled terminfo data structures. - This version first checks parsed terminfo data, then falls back to - hardcoded capabilities. + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. """ if not isinstance(cap, str): raise TypeError(f"`cap` must be a string, not {type(cap)}") - if self._capabilities: - # Fallbacks populated, use them - return self._capabilities.get(cap) - - # Look up in standard capabilities first - if cap in _STRING_CAPABILITY_NAMES: - index = _STRING_CAPABILITY_NAMES[cap] - if index < len(self._strings): - return self._strings[index] - - # Note: we don't support extended capabilities since PyREPL doesn't - # need them. - return None + return self._capabilities.get(cap) def tparm(cap_bytes: bytes, *params: int) -> bytes: From f86a59549b4b819d1eb825e5b87c9230d8b5822b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 18:58:27 +0200 Subject: [PATCH 686/713] [3.14] Pedantic rewording of why relative importing doesn't work in main modules (GH-136846) (#136940) Pedantic rewording of why relative importing doesn't work in main modules (GH-136846) Pedantically reword the section about relative imports and main modules. (cherry picked from commit 4b68289ca6954b8d135e2ee2344e67fae38239fd) Co-authored-by: Josh Cannon --- Doc/tutorial/modules.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 47bf7547b4ae1d..f8105cd5441fec 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -579,8 +579,8 @@ module for example, you might use:: from .. import formats from ..filters import equalizer -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use +Note that relative imports are based on the name of the current module's package. +Since the main module does not have a package, modules intended for use as the main module of a Python application must always use absolute imports. From aa17c393397fe582f22d3c449b2dff84ba881537 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:07:56 +0200 Subject: [PATCH 687/713] [3.14] gh-136437: Document some `os.path` functions as requiring pos-only (GH-136812) (#136944) gh-136437: Document some `os.path` functions as requiring pos-only (GH-136812) (cherry picked from commit b5428bb0e786f5b67c6077472c0068cadd0b5ea9) Co-authored-by: sobolevn --- Doc/library/os.path.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 3f2574e55699bf..734b6ef5d5d57c 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -64,7 +64,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: basename(path) +.. function:: basename(path, /) Return the base name of pathname *path*. This is the second element of the pair returned by passing *path* to the function :func:`split`. Note that @@ -237,7 +237,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isabs(path) +.. function:: isabs(path, /) Return ``True`` if *path* is an absolute pathname. On Unix, that means it begins with a slash, on Windows that it begins with two (back)slashes, or a @@ -261,7 +261,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isdir(path) +.. function:: isdir(path, /) Return ``True`` if *path* is an :func:`existing ` directory. This follows symbolic links, so both :func:`islink` and :func:`isdir` can be true @@ -372,7 +372,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object` for *path* and *paths*. -.. function:: normcase(path) +.. function:: normcase(path, /) Normalize the case of a pathname. On Windows, convert all characters in the pathname to lowercase, and also convert forward slashes to backward slashes. @@ -509,7 +509,7 @@ the :mod:`glob` module.) Added Windows support. -.. function:: split(path) +.. function:: split(path, /) Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the last pathname component and *head* is everything leading up to that. The @@ -525,7 +525,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitdrive(path) +.. function:: splitdrive(path, /) Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either a mount point or the empty string. On systems which do not use drive @@ -550,7 +550,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitroot(path) +.. function:: splitroot(path, /) Split the pathname *path* into a 3-item tuple ``(drive, root, tail)`` where *drive* is a device name or mount point, *root* is a string of separators @@ -583,7 +583,7 @@ the :mod:`glob` module.) .. versionadded:: 3.12 -.. function:: splitext(path) +.. function:: splitext(path, /) Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == path``, and the extension, *ext*, is empty or begins with a period and contains at From 81edfb7140074bac3407c175664281e7aaad046a Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:08:07 +0200 Subject: [PATCH 688/713] [3.14] GH-136874: `url2pathname()`: discard query and fragment components (GH-136875) (#136942) GH-136874: `url2pathname()`: discard query and fragment components (GH-136875) In `urllib.request.url2pathname()`, ignore any query or fragment components in the given URL. (cherry picked from commit 80b2d60a51cfd824d025eb8b3ec500acce5c010c) Co-authored-by: Barney Gale --- Doc/library/urllib.request.rst | 3 +++ Doc/whatsnew/3.14.rst | 1 + Lib/test/test_urllib.py | 8 ++++++++ Lib/urllib/request.py | 10 +++++----- .../2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst | 1 + 5 files changed, 18 insertions(+), 5 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 58bd111b5cc374..016bc777fbb232 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -210,6 +210,9 @@ The :mod:`urllib.request` module defines the following functions: Windows a UNC path is returned (as before), and on other platforms a :exc:`~urllib.error.URLError` is raised. + .. versionchanged:: 3.14 + The URL query and fragment components are discarded if present. + .. versionchanged:: 3.14 The *require_scheme* and *resolve_host* parameters were added. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index d58f7ecf02ce6b..630d011ffbbd1f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2192,6 +2192,7 @@ urllib - Discard URL authority if it matches the local hostname. - Discard URL authority if it resolves to a local IP address when the new *resolve_host* argument is set to true. + - Discard URL query and fragment components. - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 1d889ae7cf458f..c30fb5e27eea8a 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -1526,6 +1526,14 @@ def test_url2pathname(self): self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') self.assertEqual(fn('data:blah'), 'data:blah') self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') def test_url2pathname_require_scheme(self): sep = os.path.sep diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 41dc5d7b35dedb..c1c373d08815c1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1654,11 +1654,11 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False): The URL authority may be resolved with gethostbyname() if *resolve_host* is set to true. """ - if require_scheme: - scheme, url = _splittype(url) - if scheme != 'file': - raise URLError("URL is missing a 'file:' scheme") - authority, url = _splithost(url) + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") if os.name == 'nt': if not _is_local_authority(authority, resolve_host): # e.g. file://server/share/file.txt diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst new file mode 100644 index 00000000000000..9a71eb8ef1ac8d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst @@ -0,0 +1 @@ +Discard URL query and fragment in :func:`urllib.request.url2pathname`. From fa9a369044860d720387573eea5f709658146211 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 20:29:22 +0200 Subject: [PATCH 689/713] [3.14] gh-136437: Document `os.path.dirname` as accepting only pos-only (GH-136946) (#136947) gh-136437: Document `os.path.dirname` as accepting only pos-only (GH-136946) (cherry picked from commit 322442945084ea9055f86a17fa5096b11ba5b344) Co-authored-by: sobolevn --- Doc/library/os.path.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index 734b6ef5d5d57c..2ce1bd29d373ff 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -118,7 +118,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: dirname(path) +.. function:: dirname(path, /) Return the directory name of pathname *path*. This is the first element of the pair returned by passing *path* to the function :func:`split`. From 893707c53852150e65068fcf80ac5469bde0c0ee Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 21 Jul 2025 21:34:24 +0300 Subject: [PATCH 690/713] [3.14] gh-135661: Fix parsing attributes with whitespaces around the "=" separator in HTMLParser (GH-136908) (#136927) --- Lib/html/parser.py | 4 +-- Lib/test/test_htmlparser.py | 28 +++++++++++-------- Misc/NEWS.d/3.14.0b4.rst | 2 +- ...-07-21-14-15-25.gh-issue-135661.nAxXw5.rst | 2 ++ 4 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst diff --git a/Lib/html/parser.py b/Lib/html/parser.py index 9b4f09599134bd..7eea885cfe63c5 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -45,7 +45,7 @@ ( (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name ) - (= # value indicator + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator ('[^']*' # LITA-enclosed value |"[^"]*" # LIT-enclosed value |(?!['"])[^>\t\n\r\f ]* # bare value @@ -57,7 +57,7 @@ [a-zA-Z][^\t\n\r\f />]* # tag name [\t\n\r\f /]* # optional whitespace before attribute name (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name - (?:= # value indicator + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator (?:'[^']*' # LITA-enclosed value |"[^"]*" # LIT-enclosed value |(?!['"])[^>\t\n\r\f ]* # bare value diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index 15cad061889a79..47c0752fb517b9 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -623,7 +623,7 @@ def test_correct_detection_of_start_tags(self): html = '
The rain' expected = [ - ('starttag', 'div', [('style', ''), (',', None), ('foo', None), ('=', None), ('"bar"', None)]), + ('starttag', 'div', [('style', ''), (',', None), ('foo', 'bar')]), ('starttag', 'b', []), ('data', 'The '), ('starttag', 'a', [('href', 'some_url')]), @@ -813,12 +813,12 @@ def test_attr_syntax(self): ] self._run_check("""""", output) self._run_check("", [('starttag', 'a', [('foo', '=bar')])]) - self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) - self._run_check("", [('starttag', 'a', [('foo', None), ('=bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo\v', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo\xa0', 'bar')])]) - self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) - self._run_check("", [('starttag', 'a', [('foo', ''), ('bar', None)])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("", [('starttag', 'a', [('foo', 'bar')])]) self._run_check("", [('starttag', 'a', [('foo', '\vbar')])]) self._run_check("", [('starttag', 'a', [('foo', '\xa0bar')])]) @@ -829,8 +829,8 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""""", [("starttag", "a", [("b", ""), ("c", "")])]) - self._run_check("", - [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) self._run_check("", [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. @@ -899,13 +899,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), - ('starttag', 'a', [('href', None), ('=', None), ("test' style", 'color:red;bad4')]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad3'), + ('endtag', 'a'), + ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Misc/NEWS.d/3.14.0b4.rst b/Misc/NEWS.d/3.14.0b4.rst index b96f96caa3f280..349023ec75865d 100644 --- a/Misc/NEWS.d/3.14.0b4.rst +++ b/Misc/NEWS.d/3.14.0b4.rst @@ -75,7 +75,7 @@ to the HTML5 standard. * Multiple ``=`` between attribute name and value are no longer collapsed. E.g. ```` produces attribute "foo" with value "=bar". -* Whitespaces between the ``=`` separator and attribute name or value are no +* [Reverted in :gh:`136927`] Whitespaces between the ``=`` separator and attribute name or value are no longer ignored. E.g. ```` produces two attributes "foo" and "=bar", both with value None; ```` produces two attributes: "foo" with value "" and "bar" with value None. diff --git a/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst b/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst new file mode 100644 index 00000000000000..533e4df8626b90 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst @@ -0,0 +1,2 @@ +Fix parsing attributes with whitespaces around the ``=`` separator in +:class:`html.parser.HTMLParser` according to the HTML5 standard. From 718fc5b1393d9ce52fc0353fb361818e865e3e45 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Mon, 21 Jul 2025 23:01:14 +0200 Subject: [PATCH 691/713] [3.14] gh-136870: fix data race in `PyThreadState_Clear` on `sys_tracing_threads` (GH-136951) (#136953) gh-136870: fix data race in `PyThreadState_Clear` on `sys_tracing_threads` (GH-136951) In free-threading, multiple threads can be cleared concurrently as such the modifications on `sys_tracing_threads` should be done while holding the profile lock, otherwise it can race with other threads setting up profiling. (cherry picked from commit f183996eb77fd2d5662c62667298c292c943ebf5) Co-authored-by: Kumar Aditya --- Python/pystate.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Python/pystate.c b/Python/pystate.c index 015e9f8725c1af..aa2c4bd51814c3 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1793,6 +1793,10 @@ PyThreadState_Clear(PyThreadState *tstate) "PyThreadState_Clear: warning: thread still has a generator\n"); } +#ifdef Py_GIL_DISABLED + PyMutex_Lock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#endif + if (tstate->c_profilefunc != NULL) { tstate->interp->sys_profiling_threads--; tstate->c_profilefunc = NULL; @@ -1801,6 +1805,11 @@ PyThreadState_Clear(PyThreadState *tstate) tstate->interp->sys_tracing_threads--; tstate->c_tracefunc = NULL; } + +#ifdef Py_GIL_DISABLED + PyMutex_Unlock(&_PyRuntime.ceval.sys_trace_profile_mutex); +#endif + Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); From ecd97caaf5923764d22f6bd29339b8416c0c1917 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 01:23:05 +0200 Subject: [PATCH 692/713] [3.14] gh-136421: Load `_datetime` static types during interpreter initialization (GH-136583) (GH-136943) gh-136421: Load `_datetime` static types during interpreter initialization (GH-136583) `_datetime` is a special module, because it's the only non-builtin C extension that contains static types. As such, it would initialize static types in the module's execution function, which can run concurrently. Since static type initialization is not thread-safe, this caused crashes. This fixes it by moving the initialization of `_datetime`'s static types to interpreter startup (where all other static types are initialized), which is already properly protected through other locks. (cherry picked from commit a10960699a2b3e4e62896331c4f9cfd162ebf440) Co-authored-by: Peter Bierma --- Include/internal/pycore_pylifecycle.h | 1 + Lib/test/datetimetester.py | 28 +++ ...-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst | 1 + Modules/Setup.bootstrap.in | 2 + Modules/Setup.stdlib.in | 3 - Modules/_datetimemodule.c | 165 ++++++++---------- PCbuild/_freeze_module.vcxproj | 1 + Python/pylifecycle.c | 5 + 8 files changed, 111 insertions(+), 95 deletions(-) create mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6e89ca33e4208c..8faf7a4d403f84 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -41,6 +41,7 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); +extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp); /* Various internal finalizers */ diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 1b551254f86c32..1c1cbd03d02ccc 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -7275,6 +7275,34 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + def test_concurrent_initialization_subinterpreter(self): + # gh-136421: Concurrent initialization of _datetime across multiple + # interpreters wasn't thread-safe due to its static types. + + # Run in a subprocess to ensure we get a clean version of _datetime + script = """if True: + from concurrent.futures import InterpreterPoolExecutor + + def func(): + import _datetime + print('a', end='') + + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + # Now test against concurrent reinitialization + script = "import _datetime\n" + script + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst new file mode 100644 index 00000000000000..dcc73267a78546 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst @@ -0,0 +1 @@ +Fix crash when initializing :mod:`datetime` concurrently. diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 2b2e8cb3e3cacd..65a1fefe72e92e 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -12,6 +12,8 @@ posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c _suggestions _suggestions.c +# needs libm and on some platforms librt +_datetime _datetimemodule.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3a38a60a152e8c..905ea4aa2e57a9 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -56,9 +56,6 @@ @MODULE_CMATH_TRUE@cmath cmathmodule.c @MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c -# needs libm and on some platforms librt -@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c - # _decimal uses libmpdec # either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so # with ./configure --with-system-libmpdec diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index f5a9712b64e4de..c4bb44f1234990 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,7 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() +#include "pycore_initconfig.h" // _PyStatus_OK() #include "datetime.h" @@ -124,10 +125,9 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) static PyObject * -get_current_module(PyInterpreterState *interp, int *p_reloading) +get_current_module(PyInterpreterState *interp) { PyObject *mod = NULL; - int reloading = 0; PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { @@ -138,7 +138,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) goto error; } if (ref != NULL) { - reloading = 1; if (ref != Py_None) { (void)PyWeakref_GetRef(ref, &mod); if (mod == Py_None) { @@ -147,9 +146,6 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) Py_DECREF(ref); } } - if (p_reloading != NULL) { - *p_reloading = reloading; - } return mod; error: @@ -163,7 +159,7 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); + PyObject *mod = get_current_module(interp); if (mod == NULL) { assert(!PyErr_Occurred()); if (PyErr_Occurred()) { @@ -4476,7 +4472,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base; filled in PyInit__datetime */ + &PyDateTime_TZInfoType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7131,8 +7127,7 @@ static PyTypeObject PyDateTime_DateTimeType = { datetime_methods, /* tp_methods */ 0, /* tp_members */ datetime_getset, /* tp_getset */ - 0, /* tp_base; filled in - PyInit__datetime */ + &PyDateTime_DateType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7313,29 +7308,82 @@ clear_state(datetime_state *st) } -static int -init_static_types(PyInterpreterState *interp, int reloading) +PyStatus +_PyDateTime_InitTypes(PyInterpreterState *interp) { - if (reloading) { - return 0; - } - - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - /* Bases classes must be initialized before subclasses, * so capi_types must have the types in the appropriate order. */ for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; if (_PyStaticType_InitForExtension(interp, type) < 0) { - return -1; + return _PyStatus_ERR("could not initialize static types"); } } - return 0; +#define DATETIME_ADD_MACRO(dict, c, value_expr) \ + do { \ + assert(!PyErr_Occurred()); \ + PyObject *value = (value_expr); \ + if (value == NULL) { \ + goto error; \ + } \ + if (PyDict_SetItemString(dict, c, value) < 0) { \ + Py_DECREF(value); \ + goto error; \ + } \ + Py_DECREF(value); \ + } while(0) + + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } + + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ + + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); + + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO + + return _PyStatus_OK(); + +error: + return _PyStatus_NO_MEMORY(); } @@ -7353,20 +7401,15 @@ _datetime_exec(PyObject *module) { int rc = -1; datetime_state *st = get_module_state(module); - int reloading = 0; PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp, &reloading); + PyObject *old_module = get_current_module(interp); if (PyErr_Occurred()) { assert(old_module == NULL); goto error; } /* We actually set the "current" module right before a successful return. */ - if (init_static_types(interp, reloading) < 0) { - goto error; - } - for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; const char *name = _PyType_Name(type); @@ -7380,68 +7423,6 @@ _datetime_exec(PyObject *module) goto error; } -#define DATETIME_ADD_MACRO(dict, c, value_expr) \ - do { \ - assert(!PyErr_Occurred()); \ - PyObject *value = (value_expr); \ - if (value == NULL) { \ - goto error; \ - } \ - if (PyDict_SetItemString(dict, c, value) < 0) { \ - Py_DECREF(value); \ - goto error; \ - } \ - Py_DECREF(value); \ - } while(0) - - if (!reloading) { - /* timedelta values */ - PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = _PyType_GetDict(&PyDateTime_DateType); - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = _PyType_GetDict(&PyDateTime_TimeType); - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = _PyType_GetDict(&PyDateTime_DateTimeType); - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* timezone values */ - d = _PyType_GetDict(&PyDateTime_TimeZoneType); - if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { - goto error; - } - - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ - - /* -23:59 */ - DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - - /* +23:59 */ - DATETIME_ADD_MACRO( - d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); - } - -#undef DATETIME_ADD_MACRO - /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { goto error; diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index efff6a58d895cb..5ceddf759b8f3b 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -106,6 +106,7 @@ + diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c07862edf97d0b..352787c64958b2 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -760,6 +760,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyDateTime_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } From 4af1b72b31117896fd09c89063d136b51da76dbb Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 02:22:57 +0200 Subject: [PATCH 693/713] [3.14] gh-133600: Move config.site-wasm32-emscripten into the emscripten folder (GH-136934) (#136956) Reorganises the large Emscripten-specific file into the Emscripten folder. (cherry picked from commit bbe589f93ccaf32eb95fd9d1f8f3dc9a536e8db1) Co-authored-by: Hood Chatham --- Tools/wasm/emscripten/__main__.py | 2 +- Tools/wasm/{ => emscripten}/config.site-wasm32-emscripten | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename Tools/wasm/{ => emscripten}/config.site-wasm32-emscripten (97%) diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 1b31e36dd890b3..e552f6b680da9d 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -186,7 +186,7 @@ def make_emscripten_libffi(context, working_dir): def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" config_site = os.fsdecode( - CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" + EMSCRIPTEN_DIR / "config.site-wasm32-emscripten" ) emscripten_build_dir = working_dir.relative_to(CHECKOUT) diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Tools/wasm/emscripten/config.site-wasm32-emscripten similarity index 97% rename from Tools/wasm/config.site-wasm32-emscripten rename to Tools/wasm/emscripten/config.site-wasm32-emscripten index 1471546a5eec17..8c3a338dacb2dc 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Tools/wasm/emscripten/config.site-wasm32-emscripten @@ -1,6 +1,6 @@ # config.site override for cross compiling to wasm32-emscripten platform # -# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ +# CONFIG_SITE=Tools/wasm/emscripten/config.site-wasm32-emscripten \ # emconfigure ./configure --host=wasm32-unknown-emscripten --build=... # # Written by Christian Heimes From caef946a25aac557f13cb66fbc70f06225546e67 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 05:35:41 +0200 Subject: [PATCH 694/713] [3.14] gh-136170: Revert adding `ZipFile.data_offset` (GH-136950) (#136955) gh-136170: Revert adding `ZipFile.data_offset` (GH-136950) * Revert "gh-84481: Make ZipFile.data_offset more robust (GH-132178)" This reverts commit 6cd1d6c6b142697fb72f422b7b448c27ebc30534. * Revert "gh-84481: Add ZipFile.data_offset attribute (GH-132165)" This reverts commit 0788948dcb980c7648b29ca363390b696d7f188f. --------- (cherry picked from commit 6bf1c0ab3497b1b193812654bcdfd0c11b4192d8) Co-authored-by: Emma Smith Co-authored-by: Gregory P. Smith --- Doc/library/zipfile.rst | 8 --- Lib/test/test_zipfile/test_core.py | 54 ------------------- Lib/zipfile/__init__.py | 13 ----- ...-07-21-22-35-50.gh-issue-136170.QUlc78.rst | 3 ++ 4 files changed, 3 insertions(+), 75 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index bf9136a2139112..a1261ec471c92e 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -558,14 +558,6 @@ The following data attributes are also available: it should be no longer than 65535 bytes. Comments longer than this will be truncated. -.. attribute:: ZipFile.data_offset - - The offset to the start of ZIP data from the beginning of the file. When the - :class:`ZipFile` is opened in either mode ``'w'`` or ``'x'`` and the - underlying file does not support ``tell()``, the value will be ``None`` - instead. - - .. versionadded:: 3.14 .. _path-objects: diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index ada96813709aea..c033059a515db6 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -3470,60 +3470,6 @@ def test_execute_zip64(self): self.assertIn(b'number in executable: 5', output) -class TestDataOffsetPrependedZip(unittest.TestCase): - """Test .data_offset on reading zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') - - def _test_data_offset(self, name): - with zipfile.ZipFile(name) as zipfp: - self.assertEqual(zipfp.data_offset, 713) - - def test_data_offset_with_exe_prepended(self): - self._test_data_offset(self.exe_zip) - - def test_data_offset_with_exe_prepended_zip64(self): - self._test_data_offset(self.exe_zip64) - -class TestDataOffsetZipWrite(unittest.TestCase): - """Test .data_offset for ZipFile opened in write mode.""" - - def setUp(self): - os.mkdir(TESTFNDIR) - self.addCleanup(rmtree, TESTFNDIR) - self.test_path = os.path.join(TESTFNDIR, 'testoffset.zip') - - def test_data_offset_write_no_prefix(self): - with io.BytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 0) - - def test_data_offset_write_with_prefix(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_append_with_bad_zip(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "a") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_write_no_tell(self): - # The initializer in ZipFile checks if tell raises AttributeError or - # OSError when creating a file in write mode when deducing the offset - # of the beginning of zip data - class NoTellBytesIO(io.BytesIO): - def tell(self): - raise OSError("Unimplemented!") - with NoTellBytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertIsNone(zipfp.data_offset) - - class EncodedMetadataTests(unittest.TestCase): file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' file_content = [ diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 18caeb3e04a2b5..2969f735e8abb9 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -1452,7 +1452,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._lock = threading.RLock() self._seekable = True self._writing = False - self._data_offset = None try: if mode == 'r': @@ -1463,7 +1462,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._didModify = True try: self.start_dir = self.fp.tell() - self._data_offset = self.start_dir except (AttributeError, OSError): self.fp = _Tellable(self.fp) self.start_dir = 0 @@ -1488,7 +1486,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, # even if no files are added to the archive self._didModify = True self.start_dir = self.fp.tell() - self._data_offset = self.start_dir else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1535,10 +1532,6 @@ def _RealGetContents(self): # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat - # store the offset to the beginning of data for the - # .data_offset property - self._data_offset = concat - if self.start_dir < 0: raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) @@ -1599,12 +1592,6 @@ def _RealGetContents(self): zinfo._end_offset = end_offset end_offset = zinfo.header_offset - @property - def data_offset(self): - """The offset to the start of zip data in the file or None if - unavailable.""" - return self._data_offset - def namelist(self): """Return a list of file names in the archive.""" return [data.filename for data in self.filelist] diff --git a/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst b/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst new file mode 100644 index 00000000000000..fd30fe156a1b32 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst @@ -0,0 +1,3 @@ +Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in 3.14.0a7 +as it wasn't fully clear which behavior it should have in some situations so +the result was not always what a user might expect. From 6e1b31b87e7a42c7911b517b78fc418217e6480c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 07:08:15 +0200 Subject: [PATCH 695/713] [3.14] gh-135228: When @dataclass(slots=True) replaces a dataclass, make the original class collectible (GH-136893) (#136960) gh-135228: When @dataclass(slots=True) replaces a dataclass, make the original class collectible (GH-136893) An interesting hack, but more localized in scope than GH-135230. This may be a breaking change if people intentionally keep the original class around when using `@dataclass(slots=True)`, and then use `__dict__` or `__weakref__` on the original class. (cherry picked from commit 46cbdf967ada11b0286060488b61635fd6a2bb23) Co-authored-by: Jelle Zijlstra Co-authored-by: Alyssa Coghlan --- Lib/dataclasses.py | 15 ++++++++ Lib/test/test_dataclasses/__init__.py | 35 +++++++++++++++++++ ...-07-20-16-56-55.gh-issue-135228.n_XIao.rst | 4 +++ 3 files changed, 54 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 83ea623dce6281..22b78bb2fbe6ed 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1338,6 +1338,13 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): or _update_func_cell_for__class__(member.fdel, cls, newcls)): break + # gh-135228: Make sure the original class can be garbage collected. + # Bypass mapping proxy to allow __dict__ to be removed + old_cls_dict = cls.__dict__ | _deproxier + old_cls_dict.pop('__dict__', None) + if "__weakref__" in cls.__dict__: + del cls.__weakref__ + return newcls @@ -1732,3 +1739,11 @@ def _replace(self, /, **changes): # changes that aren't fields, this will correctly raise a # TypeError. return self.__class__(**changes) + + +# Hack to the get the underlying dict out of a mappingproxy +# Use it with: cls.__dict__ | _deproxier +class _Deproxier: + def __ror__(self, other): + return other +_deproxier = _Deproxier() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index e98a8f284cec9f..6bf5e5b3e5554b 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3804,6 +3804,41 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + def test_original_class_is_gced(self): + # gh-135228: Make sure when we replace the class with slots=True, the original class + # gets garbage collected. + def make_simple(): + @dataclass(slots=True) + class SlotsTest: + pass + + return SlotsTest + + def make_with_annotations(): + @dataclass(slots=True) + class SlotsTest: + x: int + + return SlotsTest + + def make_with_annotations_and_method(): + @dataclass(slots=True) + class SlotsTest: + x: int + + def method(self) -> int: + return self.x + + return SlotsTest + + for make in (make_simple, make_with_annotations, make_with_annotations_and_method): + with self.subTest(make=make): + C = make() + support.gc_collect() + candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' + and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] + self.assertEqual(candidates, [C]) + class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst new file mode 100644 index 00000000000000..ee8962c6f46e75 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst @@ -0,0 +1,4 @@ +When :mod:`dataclasses` replaces a class with a slotted dataclass, the +original class is now garbage collected again. Earlier changes in Python +3.14 caused this class to remain in existence together with the replacement +class synthesized by :mod:`dataclasses`. From efa984b7ce1b6c544e445adcca806ab5fd84de6c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:15:50 +0200 Subject: [PATCH 696/713] [3.14] gh-131531: Android test fixes (GH-136845) (#136962) Modifies the test runner script to no longer export the the HOST environment variable, and to allow for tests that produce no Python output (output from the Android console is still expected and required). These changes stem from knowledge gained during developing a PR for Android support in cibuildwheel. (cherry picked from commit 149bddcc216a398d71ec9497e9bf3ec03d6f2914) Co-authored-by: Malcolm Smith --- Android/android.py | 73 +++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/Android/android.py b/Android/android.py index a3a48c0c6b7027..75f73cd30993da 100755 --- a/Android/android.py +++ b/Android/android.py @@ -50,7 +50,19 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -118,7 +130,7 @@ def android_env(host): env_script = ANDROID_DIR / "android-env.sh" env_output = subprocess.run( f"set -eu; " - f"export HOST={host}; " + f"HOST={host}; " f"PREFIX={prefix}; " f". {env_script}; " f"export", @@ -453,17 +465,19 @@ async def logcat_task(context, initial_devices): # `--pid` requires API level 24 or higher. args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + logcat_started = False async with async_process( *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): + logcat_started = True level, message = match.groups() else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. + # If the regex doesn't match, this is either a logcat startup + # error, or the second or subsequent line of a multi-line + # message. Python won't produce multi-line messages, but other + # components might. level, message = None, line # Exclude high-volume messages which are rarely useful. @@ -483,25 +497,22 @@ async def logcat_task(context, initial_devices): # tag indicators from Python's stdout and stderr. for prefix in ["python.stdout: ", "python.stderr: "]: if message.startswith(prefix): - global logcat_started - logcat_started = True + global python_started + python_started = True stream.write(message.removeprefix(prefix)) break else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. + log_verbose(context, line, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) def stop_app(serial): @@ -516,16 +527,6 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected - hidden_output = [] - - def log(line): - # Gradle may take several minutes to install SDK packages, so it's worth - # showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): - sys.stdout.write(line) - else: - hidden_output.append(line) - if context.command: mode = "-c" module = context.command @@ -550,7 +551,7 @@ def log(line): ] if context.verbose >= 2: args.append("--info") - log("> " + join_command(args)) + log_verbose(context, f"> {join_command(args)}\n") try: async with async_process( @@ -558,7 +559,12 @@ def log(line): stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) as process: while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - log(line) + # Gradle may take several minutes to install SDK packages, so + # it's worth showing those messages even in non-verbose mode. + if line.startswith('Preparing "Install'): + sys.stdout.write(line) + else: + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -566,11 +572,6 @@ def log(line): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) @@ -600,6 +601,12 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] From 72b679423da7fbb4fd4d238d2ade81ca698dbdbc Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:29:17 +0300 Subject: [PATCH 697/713] [3.14] Fix 3 typos in "Next" News items (GH-136892) (#136967) Co-authored-by: Cornelius Roemer --- Misc/NEWS.d/3.14.0b1.rst | 4 ++-- .../2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 5847dea7d5e8ee..f76896954f321d 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -1756,7 +1756,7 @@ Add support for macOS multi-arch builds with the JIT enabled .. nonce: q9fvyM .. section: Core and Builtins -PyREPL now supports syntax highlighing. Contributed by Łukasz Langa. +PyREPL now supports syntax highlighting. Contributed by Łukasz Langa. .. @@ -1797,7 +1797,7 @@ non-``None`` ``closure``. Patch by Bartosz Sławecki. .. nonce: Uj7lyY .. section: Core and Builtins -Fix a bug that was allowing newlines inconsitently in format specifiers for +Fix a bug that was allowing newlines inconsistently in format specifiers for single-quoted f-strings. Patch by Pablo Galindo. .. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst index 5c0813b1a0abda..767d7b97726971 100644 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst +++ b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst @@ -1 +1 @@ -Fix PyREPL syntax highlightning on match cases after multi-line case. Contributed by Olga Matoula. +Fix PyREPL syntax highlighting on match cases after multi-line case. Contributed by Olga Matoula. From d5d3adafa71278da574533f30325605d67a1cdec Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:29:40 +0200 Subject: [PATCH 698/713] [3.14] gh-135468: Improve ``BaseHandler.http_error_default()`` parameter descriptions (GH-136797) (#136825) Co-authored-by: Valerio Gianella <49408327+valeriogianella@users.noreply.github.com> Co-authored-by: Adam Turner <9087854+aa-turner@users.noreply.github.com> --- Doc/library/urllib.request.rst | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 016bc777fbb232..e514b98fc5d553 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -832,10 +832,13 @@ The following attribute and methods should only be used by classes derived from errors. It will be called automatically by the :class:`OpenerDirector` getting the error, and should not normally be called in other circumstances. - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. + :class:`OpenerDirector` will call this method with five positional arguments: + + 1. a :class:`Request` object, + #. a file-like object with the HTTP error body, + #. the three-digit code of the error, as a string, + #. the user-visible explanation of the code, as as string, and + #. the headers of the error, as a mapping object. Return values and exceptions raised should be the same as those of :func:`urlopen`. From ddd3413687ef411c7bf7b029337cdf48c6d6c59c Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 10:32:09 +0200 Subject: [PATCH 699/713] [3.14] gh-136859: Improve `StrEnum` docs (GH-136864) (#136936) Co-authored-by: Nacho Caballero Co-authored-by: Nacho Caballero Co-authored-by: Antonio Spadaro --- Doc/library/enum.rst | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c9b2c7d76b6746..2cfc2f4962979f 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -504,16 +504,31 @@ Data Types .. class:: StrEnum - ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used - in most of the same places that a string can be used. The result of any string - operation performed on or with a *StrEnum* member is not part of the enumeration. + *StrEnum* is the same as :class:`Enum`, but its members are also strings and + can be used in most of the same places that a string can be used. The result + of any string operation performed on or with a *StrEnum* member is not part + of the enumeration. + + >>> from enum import StrEnum, auto + >>> class Color(StrEnum): + ... RED = 'r' + ... GREEN = 'g' + ... BLUE = 'b' + ... UNKNOWN = auto() + ... + >>> Color.RED + + >>> Color.UNKNOWN + + >>> str(Color.UNKNOWN) + 'unknown' .. note:: There are places in the stdlib that check for an exact :class:`str` instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` instead of ``isinstance(unknown, str)``), and in those locations you - will need to use ``str(StrEnum.member)``. + will need to use ``str(MyStrEnum.MY_MEMBER)``. .. note:: From 11f510167c5a87e12e9277e36a12841b6b6993b3 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:44:59 +0200 Subject: [PATCH 700/713] [3.14] gh-133296: Publicly expose critical section API that accepts PyMutex (gh-135899) (#136969) Co-authored-by: Nathan Goldbaum Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Doc/c-api/init.rst | 40 +++++++++++++++++++ Include/cpython/critical_section.h | 20 ++++++++++ Include/internal/pycore_critical_section.h | 14 +------ ...-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst | 3 ++ Modules/_ctypes/ctypes.h | 2 +- Modules/_testcapimodule.c | 10 +++++ Objects/typeobject.c | 4 +- Python/critical_section.c | 18 +++++++++ 8 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index dc96ed7f719fcf..51e9e6c352f0e4 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -2458,6 +2458,12 @@ is resumed, and its locks reacquired. This means the critical section API provides weaker guarantees than traditional locks -- they are useful because their behavior is similar to the :term:`GIL`. +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + The functions and structs used by the macros are exposed for cases where C macros are not available. They should only be used as in the given macro expansions. Note that the sizes and contents of the structures may @@ -2503,6 +2509,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION() Ends the critical section and releases the per-object lock. @@ -2532,6 +2555,23 @@ code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. .. versionadded:: 3.13 +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + .. c:macro:: Py_END_CRITICAL_SECTION2() Ends the critical section and releases the per-object locks. diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 35db3fb6a59ce6..4fc46fefb93a24 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -73,22 +73,32 @@ typedef struct PyCriticalSection2 PyCriticalSection2; PyAPI_FUNC(void) PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); +PyAPI_FUNC(void) +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + PyAPI_FUNC(void) PyCriticalSection_End(PyCriticalSection *c); PyAPI_FUNC(void) PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); +PyAPI_FUNC(void) +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + PyAPI_FUNC(void) PyCriticalSection2_End(PyCriticalSection2 *c); #ifndef Py_GIL_DISABLED # define Py_BEGIN_CRITICAL_SECTION(op) \ { +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { # define Py_END_CRITICAL_SECTION() \ } # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { # define Py_END_CRITICAL_SECTION2() \ } #else /* !Py_GIL_DISABLED */ @@ -118,6 +128,11 @@ struct PyCriticalSection2 { PyCriticalSection _py_cs; \ PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_BeginMutex(&_py_cs, mutex) + # define Py_END_CRITICAL_SECTION() \ PyCriticalSection_End(&_py_cs); \ } @@ -127,6 +142,11 @@ struct PyCriticalSection2 { PyCriticalSection2 _py_cs2; \ PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + # define Py_END_CRITICAL_SECTION2() \ PyCriticalSection2_End(&_py_cs2); \ } diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 42f06b935bd0a0..2b49b9f00df3ea 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -21,16 +21,6 @@ extern "C" { #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ - { \ - PyCriticalSection _py_cs; \ - _PyCriticalSection_BeginMutex(&_py_cs, mutex) - -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ - { \ - PyCriticalSection2 _py_cs2; \ - _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - // Specialized version of critical section locking to safely use // PySequence_Fast APIs without the GIL. For performance, the argument *to* // PySequence_Fast() is provided to the macro, not the *result* of @@ -75,8 +65,6 @@ extern "C" { #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -119,6 +107,7 @@ _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) _PyCriticalSection_BeginSlow(c, m); } } +#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void _PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) @@ -194,6 +183,7 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) _PyCriticalSection2_BeginSlow(c, m1, m2, 0); } } +#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) diff --git a/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst b/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst new file mode 100644 index 00000000000000..41401911a6bc0b --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst @@ -0,0 +1,3 @@ +New variants for the critical section API that accept one or two +:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now +public in the non-limited C API. diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 03fde30c2e0944..f8389f98fec780 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -419,7 +419,7 @@ typedef struct { visible to other threads before the `dict_final` bit is set. */ -#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) +#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUTEX(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() static inline uint8_t diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 71fffedee146fa..2572df9719a703 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -2419,6 +2419,16 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) Py_BEGIN_CRITICAL_SECTION2(module, module); Py_END_CRITICAL_SECTION2(); +#ifdef Py_GIL_DISABLED + // avoid unused variable compiler warning on GIL-enabled build + PyMutex mut = {0}; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&mut); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2_MUTEX(&mut, &mut); + Py_END_CRITICAL_SECTION2(); +#endif + Py_RETURN_NONE; } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a348ea1d531c93..ab6ad1fe8b5f91 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -65,11 +65,11 @@ class object "PyObject *" "&PyBaseObject_Type" // be released and reacquired during a subclass update if there's contention // on the subclass lock. #define TYPE_LOCK &PyInterpreterState_Get()->types.mutex -#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK) #define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() #define BEGIN_TYPE_DICT_LOCK(d) \ - Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() diff --git a/Python/critical_section.c b/Python/critical_section.c index 73857b85496316..e628ba2f6d19bc 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -130,6 +130,15 @@ PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) #endif } +#undef PyCriticalSection_BeginMutex +void +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_BeginMutex(c, m); +#endif +} + #undef PyCriticalSection_End void PyCriticalSection_End(PyCriticalSection *c) @@ -148,6 +157,15 @@ PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) #endif } +#undef PyCriticalSection2_BeginMutex +void +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_BeginMutex(c, m1, m2); +#endif +} + #undef PyCriticalSection2_End void PyCriticalSection2_End(PyCriticalSection2 *c) From 8e43b130f7b27657a8bb775e398c0dce325b4741 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 22 Jul 2025 12:48:08 +0300 Subject: [PATCH 701/713] [3.14] gh-134009: Expose `PyMutex_IsLocked` in the public C API (gh-134365) (#136971) Co-authored-by: Sam Gross --- Doc/c-api/init.rst | 12 ++++++++++++ Doc/whatsnew/3.14.rst | 1 + Include/cpython/lock.h | 11 +++++++++++ Include/internal/pycore_lock.h | 7 ------- .../2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst | 1 + Python/lock.c | 8 ++++++++ 6 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 51e9e6c352f0e4..2e255a8781c353 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -2441,6 +2441,18 @@ The C-API provides a basic mutual exclusion lock. .. versionadded:: 3.13 +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + .. _python-critical-section-api: Python Critical Section API diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 630d011ffbbd1f..2906828e05d229 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -3027,6 +3027,7 @@ Porting to Python 3.14 * ``_PyLong_IsPositive()``: :c:func:`PyLong_IsPositive` * ``_PyLong_IsZero()``: :c:func:`PyLong_IsZero` * ``_PyLong_Sign()``: :c:func:`PyLong_GetSign` + * ``_PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` * ``_PyUnicodeWriter_Dealloc()``: :c:func:`PyUnicodeWriter_Discard` * ``_PyUnicodeWriter_Finish()``: :c:func:`PyUnicodeWriter_Finish` * ``_PyUnicodeWriter_Init()``: use :c:func:`PyUnicodeWriter_Create` diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h index 8ee03e82f74dfd..63886fca28eae2 100644 --- a/Include/cpython/lock.h +++ b/Include/cpython/lock.h @@ -36,6 +36,9 @@ PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); // exported function for unlocking the mutex PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); +// exported function for checking if the mutex is locked +PyAPI_FUNC(int) PyMutex_IsLocked(PyMutex *m); + // Locks the mutex. // // If the mutex is currently locked, the calling thread will be parked until @@ -61,3 +64,11 @@ _PyMutex_Unlock(PyMutex *m) } } #define PyMutex_Unlock _PyMutex_Unlock + +// Checks if the mutex is currently locked. +static inline int +_PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; +} +#define PyMutex_IsLocked _PyMutex_IsLocked diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 7484b05d7f2446..9b071573ad3c74 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -25,13 +25,6 @@ PyMutex_LockFast(PyMutex *m) return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Checks if the mutex is currently locked. -static inline int -PyMutex_IsLocked(PyMutex *m) -{ - return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; -} - // Re-initializes the mutex after a fork to the unlocked state. static inline void _PyMutex_at_fork_reinit(PyMutex *m) diff --git a/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst b/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst new file mode 100644 index 00000000000000..f060f09de19bb8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst @@ -0,0 +1 @@ +Expose :c:func:`PyMutex_IsLocked` as part of the public C API. diff --git a/Python/lock.c b/Python/lock.c index 3c0804c468e08d..ca269bd2cb4b51 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -619,3 +619,11 @@ PyMutex_Unlock(PyMutex *m) Py_FatalError("unlocking mutex that is not locked"); } } + + +#undef PyMutex_IsLocked +int +PyMutex_IsLocked(PyMutex *m) +{ + return _PyMutex_IsLocked(m); +} From d118bc061b4f7ea916bb25f19c5c7f8e1923fbc7 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 11:51:02 +0200 Subject: [PATCH 702/713] [3.14] gh-124621: Emscripten: Add support for async input devices (GH-136822) (GH-136935) This is useful for implementing proper `input()`. It requires the JavaScript engine to support the wasm JSPI spec which is now stage 4. It is supported on Chrome since version 137 and on Firefox and node behind a flag. We override the `__wasi_fd_read()` syscall with our own variant that checks for a readAsync operation. If it has it, we use our own async variant of `fd_read()`, otherwise we use the original `fd_read()`. We also add a variant of `FS.createDevice()` called `FS.createAsyncInputDevice()`. Finally, if JSPI is available, we wrap the `main()` symbol with `WebAssembly.promising()` so that we can stack switch from `fd_read()`. If JSPI is not available, attempting to read from an AsyncInputDevice will raise an `OSError`. (cherry picked from commit 7ae4749d064bd49b0dd96172fee20c1f1678d9e9) Co-authored-by: Hood Chatham --- Lib/test/test_capi/test_emscripten.py | 25 ++++ Modules/_testinternalcapi.c | 34 +++++ Python/emscripten_syscalls.c | 182 ++++++++++++++++++++++++++ Tools/wasm/emscripten/__main__.py | 12 +- 4 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 Lib/test/test_capi/test_emscripten.py diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py new file mode 100644 index 00000000000000..272d9a10ceb950 --- /dev/null +++ b/Lib/test/test_capi/test_emscripten.py @@ -0,0 +1,25 @@ +import unittest +from test.support import is_emscripten + +if not is_emscripten: + raise unittest.SkipTest("Emscripten-only test") + +from _testinternalcapi import emscripten_set_up_async_input_device +from pathlib import Path + + +class EmscriptenAsyncInputDeviceTest(unittest.TestCase): + def test_emscripten_async_input_device(self): + jspi_supported = emscripten_set_up_async_input_device() + p = Path("/dev/blah") + self.addCleanup(p.unlink) + if not jspi_supported: + with open(p, "r") as f: + self.assertRaises(OSError, f.readline) + return + + with open(p, "r") as f: + for _ in range(10): + self.assertEqual(f.readline().strip(), "ab") + self.assertEqual(f.readline().strip(), "fi") + self.assertEqual(f.readline().strip(), "xy") diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 8027f0015c7409..f84cf1a4263a2d 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -2346,6 +2346,37 @@ incref_decref_delayed(PyObject *self, PyObject *op) Py_RETURN_NONE; } +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" + +EM_JS(int, emscripten_set_up_async_input_device_js, (void), { + let idx = 0; + const encoder = new TextEncoder(); + const bufs = [ + encoder.encode("ab\n"), + encoder.encode("fi\n"), + encoder.encode("xy\n"), + ]; + function sleep(t) { + return new Promise(res => setTimeout(res, t)); + } + FS.createAsyncInputDevice("/dev", "blah", async () => { + await sleep(5); + return bufs[(idx ++) % 3]; + }); + return !!WebAssembly.promising; +}); + +static PyObject * +emscripten_set_up_async_input_device(PyObject *self, PyObject *Py_UNUSED(ignored)) { + if (emscripten_set_up_async_input_device_js()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +#endif + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -2448,6 +2479,9 @@ static PyMethodDef module_functions[] = { {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF +#ifdef __EMSCRIPTEN__ + {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index bb80f979420ec1..886262acbc6810 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -37,3 +37,185 @@ EM_JS(int, __syscall_umask_js, (int mask), { int __syscall_umask(int mask) { return __syscall_umask_js(mask); } + +#include +#include +#undef errno + +// Variant of EM_JS that does C preprocessor substitution on the body +#define EM_JS_MACROS(ret, func_name, args, body...) \ + EM_JS(ret, func_name, args, body) + +EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { + // Define FS.createAsyncInputDevice(), This is quite similar to + // FS.createDevice() defined here: + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 + // but instead of returning one byte at a time, the input() function should + // return a Uint8Array. This makes the handler code simpler, the + // `createAsyncInputDevice` simpler, and everything faster. + FS.createAsyncInputDevice = function(parent, name, input) { + parent = typeof parent == 'string' ? parent : FS.getPath(parent); + var path = PATH.join2(parent, name); + var mode = FS_getMode(true, false); + FS.createDevice.major ||= 64; + var dev = FS.makedev(FS.createDevice.major++, 0); + async function getDataBuf() { + var buf; + try { + buf = await input(); + } catch (e) { + throw new FS.ErrnoError(EIO); + } + if (!buf?.byteLength) { + throw new FS.ErrnoError(EAGAIN); + } + ops._dataBuf = buf; + } + + var ops = { + _dataBuf: new Uint8Array(0), + open(stream) { + stream.seekable = false; + }, + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + buffer = buffer.subarray(offset, offset + length); + if (!ops._dataBuf.byteLength) { + await getDataBuf(); + } + var toRead = Math.min(ops._dataBuf.byteLength, buffer.byteLength); + buffer.subarray(0, toRead).set(ops._dataBuf); + buffer = buffer.subarray(toRead); + ops._dataBuf = ops._dataBuf.subarray(toRead); + if (toRead) { + stream.node.atime = Date.now(); + } + return toRead; + }, + }; + FS.registerDevice(dev, ops); + return FS.mkdev(path, mode, dev); + }; + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + const origResolveGlobalSymbol = resolveGlobalSymbol; + if (!Module.onExit && process?.exit) { + Module.onExit = (code) => process.exit(code); + } + // * wrap the main symbol with WebAssembly.promising, + // * call exit_with_live_runtime() to prevent emscripten from shutting down + // the runtime before the promise resolves, + // * call onExit / process.exit ourselves, since exit_with_live_runtime() + // prevented Emscripten from calling it normally. + resolveGlobalSymbol = function (name, direct = false) { + const orig = origResolveGlobalSymbol(name, direct); + if (name === "main") { + const main = WebAssembly.promising(orig.sym); + orig.sym = (...args) => { + (async () => { + const ret = await main(...args); + process?.exit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + } + return orig; + }; +}) + +__attribute__((constructor)) void _emscripten_promising_main(void) { + _emscripten_promising_main_js(); +} + + +#define IOVEC_T_BUF_OFFSET 0 +#define IOVEC_T_BUF_LEN_OFFSET 4 +#define IOVEC_T_SIZE 8 +_Static_assert(offsetof(__wasi_iovec_t, buf) == IOVEC_T_BUF_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(offsetof(__wasi_iovec_t, buf_len) == IOVEC_T_BUF_LEN_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, + "Unexpected __wasi_iovec_t layout"); + +// If the stream has a readAsync handler, read to buffer defined in iovs, write +// number of bytes read to *nread, and return a promise that resolves to the +// errno. Otherwise, return null. +EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( + __wasi_fd_t fd, + const __wasi_iovec_t *iovs, + size_t iovcnt, + __wasi_size_t *nread +), { + if (!WebAssembly.promising) { + return null; + } + var stream; + try { + stream = SYSCALLS.getStreamFromFD(fd); + } catch (e) { + // If the fd was already closed or never existed, getStreamFromFD() + // raises. We'll let fd_read_orig() handle setting errno. + return null; + } + if (!stream.stream_ops.readAsync) { + // Not an async device. Fall back to __wasi_fd_read_orig(). + return null; + } + return (async () => { + // This is the same as libwasi.js fd_read() and doReadv() except we use + // readAsync and we await it. + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L331 + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L197 + try { + var ret = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[(iovs + IOVEC_T_BUF_OFFSET)/4]; + var len = HEAP32[(iovs + IOVEC_T_BUF_LEN_OFFSET)/4]; + iovs += IOVEC_T_SIZE; + var curr = await stream.stream_ops.readAsync(stream, HEAP8, ptr, len); + if (curr < 0) return -1; + ret += curr; + if (curr < len) break; // nothing more to read + } + HEAP32[nread/4] = ret; + return 0; + } catch (e) { + if (e.name !== 'ErrnoError') { + throw e; + } + return e.errno; + } + })(); +}; +); + +// Bind original fd_read syscall to __wasi_fd_read_orig(). +__wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("fd_read"), __warn_unused_result__)); + +// Take a promise that resolves to __wasi_errno_t and suspend until it resolves, +// get the output. +EM_JS(__wasi_errno_t, __block_for_errno, (__externref_t p), { + return p; +} +if (WebAssembly.Suspending) { + __block_for_errno = new WebAssembly.Suspending(__block_for_errno); +} +) + +// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_errno +// to get the result. +__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) { + __externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread); + if (__builtin_wasm_ref_is_null_extern(p)) { + return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); + } + __wasi_errno_t res = __block_for_errno(p); + return res; +} diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index e552f6b680da9d..b25cbb01dedd31 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -274,10 +274,20 @@ def configure_emscripten_python(context, working_dir): REALPATH=abs_path fi + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) From c59a60bdb48120fe7b7ba65507448db358b6671d Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 12:32:00 +0200 Subject: [PATCH 703/713] [3.14] gh-132661: Document t-strings and `templatelib` (GH-135229) (#136974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Dave Peck Co-authored-by: Petr Viktorin Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Co-authored-by: Loïc Simon Co-authored-by: pauleveritt Co-authored-by: Lysandros Nikolaou --- Doc/glossary.rst | 7 +- Doc/library/ast.rst | 54 ++++- Doc/library/dis.rst | 42 ++++ Doc/library/stdtypes.rst | 6 +- Doc/library/string.rst | 25 ++- Doc/library/string.templatelib.rst | 313 +++++++++++++++++++++++++++++ Doc/library/text.rst | 1 + Doc/reference/compound_stmts.rst | 4 +- Doc/reference/lexical_analysis.rst | 50 ++++- Doc/tutorial/inputoutput.rst | 9 +- 10 files changed, 488 insertions(+), 23 deletions(-) create mode 100644 Doc/library/string.templatelib.rst diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 199a917f9f101e..b7bd547d38fd1e 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -462,7 +462,7 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -1322,6 +1322,11 @@ Glossary See also :term:`borrowed reference`. + t-string + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ca0654acb33689..eff093eebc158e 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -290,9 +290,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 115 (``ord('s')``): ``!s`` string formatting + * 114 (``ord('r')``): ``!r`` repr formatting + * 97 (``ord('a')``): ``!a`` ASCII formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -326,6 +326,54 @@ Literals Constant(value='.3')]))])) +.. class:: TemplateStr(values) + + A t-string, comprising a series of :class:`Interpolation` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('t"{name} finished {place:ordinal}"', mode='eval'), indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name', ctx=Load()), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place', ctx=Load()), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + + .. versionadded:: 3.14 + + +.. class:: Interpolation(value, str, conversion, format_spec) + + Node representing a single interpolation field in a t-string. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``str`` is a constant containing the text of the interpolation expression. + * ``conversion`` is an integer: + + * -1: no conversion + * 115: ``!s`` string conversion + * 114: ``!r`` repr conversion + * 97: ``!a`` ascii conversion + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + .. versionadded:: 3.14 + + .. class:: List(elts, ctx) Tuple(elts, ctx) diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 11685a32f48e4f..ac8a911c40a860 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -1120,6 +1120,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` from a tuple + of strings and a tuple of interpolations and pushes the resulting instance + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` from a + value and its source expression and pushes the resulting instance onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 8e688f03eb3a87..90683c0b00d78a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2675,9 +2675,9 @@ For example: lead to a number of common errors (such as failing to display tuples and dictionaries correctly). Using the newer :ref:`formatted string literals `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. + ($-strings) ` may help avoid these errors. + Each of these alternatives provides their own trade-offs and benefits of + simplicity, flexibility, and/or extensibility. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. diff --git a/Doc/library/string.rst b/Doc/library/string.rst index 23e15780075435..83e8ee2722ed8a 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -198,8 +198,9 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. +related to that of :ref:`formatted string literals ` and +:ref:`template string literals `, but it is less sophisticated +and, in particular, does not support arbitrary expressions. .. index:: single: {} (curly brackets); in string formatting @@ -264,6 +265,8 @@ Some simple format string examples:: "Weight in tons {0.weight}" # 'weight' attribute of first positional arg "Units destroyed: {players[0]}" # First element of keyword argument 'players'. +.. _formatstrings-conversion: + The *conversion* field causes a type coercion before formatting. Normally, the job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted @@ -306,7 +309,7 @@ Format Specification Mini-Language "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). +:ref:`formatstrings`, :ref:`f-strings`, and :ref:`t-strings`). They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -789,10 +792,20 @@ Nesting arguments and more complex examples:: -.. _template-strings: +.. _template-strings-pep292: -Template strings ----------------- +Template strings ($-strings) +---------------------------- + +.. note:: + + The feature described here was introduced in Python 2.4. It is unrelated + to, and should not be confused with, the newer + :ref:`template strings ` and + :ref:`t-string literal syntax ` introduced in Python 3.14. + T-string literals evaluate to instances of a different + :class:`~string.templatelib.Template` class, found in the + :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst new file mode 100644 index 00000000000000..31b90d75f411f0 --- /dev/null +++ b/Doc/library/string.templatelib.rst @@ -0,0 +1,313 @@ +:mod:`!string.templatelib` --- Support for template string literals +=================================================================== + +.. module:: string.templatelib + :synopsis: Support for template string literals. + +**Source code:** :source:`Lib/string/templatelib.py` + +-------------- + +.. seealso:: + + * :ref:`Format strings ` + * :ref:`T-string literal syntax ` + + +.. _template-strings: + +Template strings +---------------- + +.. versionadded:: 3.14 + +Template strings are a formatting mechanism that allows for deep control over +how strings are processed. You can create templates using +:ref:`t-string literal syntax `, which is identical to +:ref:`f-string syntax ` but uses a ``t`` instead of an ``f``. +While f-strings evaluate to ``str``, t-strings create a :class:`Template` +instance that gives you access to the static and interpolated (in curly braces) +parts of a string *before* they are combined. + + +.. _templatelib-template: + +Template +-------- + +The :class:`!Template` class describes the contents of a template string. + +:class:`!Template` instances are immutable: their attributes cannot be +reassigned. + +.. class:: Template(*args) + + Create a new :class:`!Template` object. + + :param args: A mix of strings and :class:`Interpolation` instances in any order. + :type args: str | Interpolation + + The most common way to create a :class:`!Template` instance is to use the + :ref:`t-string literal syntax `. This syntax is identical to that of + :ref:`f-strings ` except that it uses a ``t`` instead of an ``f``: + + >>> name = "World" + >>> template = t"Hello {name}!" + >>> type(template) + + + Templates ars stored as sequences of literal :attr:`~Template.strings` + and dynamic :attr:`~Template.interpolations`. + A :attr:`~Template.values` attribute holds the interpolation values: + + >>> template.strings + ('Hello ', '!') + >>> template.interpolations + (Interpolation('World', ...),) + >>> template.values + ('World',) + + The :attr:`!strings` tuple has one more element than :attr:`!interpolations` + and :attr:`!values`; the interpolations “belong” between the strings. + This may be easier to understand when tuples are aligned:: + + template.strings: ('Hello ', '!') + template.values: ( 'World', ) + + While literal syntax is the most common way to create :class:`!Template` + instances, it is also possible to create them directly using the constructor: + + >>> from string.templatelib import Interpolation, Template + >>> name = "World" + >>> template = Template("Hello, ", Interpolation(name, "name"), "!") + >>> list(template) + ['Hello, ', Interpolation('World', 'name', None, ''), '!'] + + If two or more consecutive strings are passed, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: + + >>> from string.templatelib import Template + >>> template = Template("Hello ", "World", "!") + >>> template.strings + ('Hello World!',) + + If two or more consecutive interpolations are passed, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: + + >>> from string.templatelib import Interpolation, Template + >>> template = Template(Interpolation("World", "name"), Interpolation("!", "punctuation")) + >>> template.strings + ('', '', '') + + .. attribute:: strings + :type: tuple[str, ...] + + A :ref:`tuple ` of the static strings in the template. + + >>> name = "World" + >>> t"Hello {name}!".strings + ('Hello ', '!') + + Empty strings *are* included in the tuple: + + >>> name = "World" + >>> t"Hello {name}{name}!".strings + ('Hello ', '', '!') + + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: + + >>> t"".strings + ('',) + >>> t"".values + () + >>> t"{'cheese'}".strings + ('', '') + >>> t"{'cheese'}".values + ('cheese',) + + .. attribute:: interpolations + :type: tuple[Interpolation, ...] + + A tuple of the interpolations in the template. + + >>> name = "World" + >>> t"Hello {name}!".interpolations + (Interpolation('World', 'name', None, ''),) + + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: + + >>> t"Hello!".interpolations + () + + .. attribute:: values + :type: tuple[Any, ...] + + A tuple of all interpolated values in the template. + + >>> name = "World" + >>> t"Hello {name}!".values + ('World',) + + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is equivalent to + ``tuple(i.value for i in template.interpolations)``. + + .. describe:: iter(template) + + Iterate over the template, yielding each string and + :class:`Interpolation` in order. + + >>> name = "World" + >>> list(t"Hello {name}!") + ['Hello ', Interpolation('World', 'name', None, ''), '!'] + + Empty strings are *not* included in the iteration: + + >>> name = "World" + >>> list(t"Hello {name}{name}") + ['Hello ', Interpolation('World', 'name', None, ''), Interpolation('World', 'name', None, '')] + + .. describe:: template + other + template += other + + Concatenate this template with another, returning a new + :class:`!Template` instance: + + >>> name = "World" + >>> list(t"Hello " + t"there {name}!") + ['Hello there ', Interpolation('World', 'name', None, ''), '!'] + + Concatenation between a :class:`!Template` and a ``str`` is *not* supported. + This is because it is ambiguous whether the string should be treated as + a static string or an interpolation. If you want to concatenate a + :class:`!Template` with a string, you should either wrap the string + directly in a :class:`!Template` (to treat it as a static string) or use + an :class:`!Interpolation` (to treat it as dynamic): + + >>> from string.templatelib import Template, Interpolation + >>> template = t"Hello " + >>> # Treat "there " as a static string + >>> template += Template("there ") + >>> # Treat name as an interpolation + >>> name = "World" + >>> template += Template(Interpolation(name, "name")) + >>> list(template) + ['Hello there ', Interpolation('World', 'name', None, '')] + + +.. class:: Interpolation(value, expression="", conversion=None, format_spec="") + + Create a new :class:`!Interpolation` object. + + :param value: The evaluated, in-scope result of the interpolation. + :type value: object + + :param expression: The text of a valid Python expression, or an empty string. + :type expression: str + + :param conversion: The optional :ref:`conversion ` to be used, one of r, s, and a. + :type conversion: ``Literal["a", "r", "s"] | None`` + + :param format_spec: An optional, arbitrary string used as the :ref:`format specification ` to present the value. + :type format_spec: str + + The :class:`!Interpolation` type represents an expression inside a template string. + + :class:`!Interpolation` instances are immutable: their attributes cannot be + reassigned. + + .. attribute:: value + + :returns: The evaluated value of the interpolation. + :type: object + + >>> t"{1 + 2}".interpolations[0].value + 3 + + .. attribute:: expression + + :returns: The text of a valid Python expression, or an empty string. + :type: str + + The :attr:`~Interpolation.expression` is the original text of the + interpolation's Python expression, if the interpolation was created + from a t-string literal. Developers creating interpolations manually + should either set this to an empty string or choose a suitable valid + Python expression. + + >>> t"{1 + 2}".interpolations[0].expression + '1 + 2' + + .. attribute:: conversion + + :returns: The conversion to apply to the value, or ``None``. + :type: ``Literal["a", "r", "s"] | None`` + + The :attr:`!Interpolation.conversion` is the optional conversion to apply + to the value: + + >>> t"{1 + 2!a}".interpolations[0].conversion + 'a' + + .. note:: + + Unlike f-strings, where conversions are applied automatically, + the expected behavior with t-strings is that code that *processes* the + :class:`!Template` will decide how to interpret and whether to apply + the :attr:`!Interpolation.conversion`. + + .. attribute:: format_spec + + :returns: The format specification to apply to the value. + :type: str + + The :attr:`!Interpolation.format_spec` is an optional, arbitrary string + used as the format specification to present the value: + + >>> t"{1 + 2:.2f}".interpolations[0].format_spec + '.2f' + + .. note:: + + Unlike f-strings, where format specifications are applied automatically + via the :func:`format` protocol, the expected behavior with + t-strings is that code that *processes* the :class:`!Template` will + decide how to interpret and whether to apply the format specification. + As a result, :attr:`!Interpolation.format_spec` values in + :class:`!Template` instances can be arbitrary strings, even those that + do not necessarily conform to the rules of Python's :func:`format` + protocol. + + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = Interpolation(3.0, "1 + 2", None, ".2f") + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec) + ... + 3.0 1 + 2 None .2f + + +Helper functions +---------------- + +.. function:: convert(obj, /, conversion) + + Applies formatted string literal :ref:`conversion ` + semantics to the given object *obj*. + This is frequently useful for custom template string processing logic. + + Three conversion flags are currently supported: + + * ``'s'`` which calls :func:`str` on the value, + * ``'r'`` which calls :func:`repr`, and + * ``'a'`` which calls :func:`ascii`. + + If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/text.rst b/Doc/library/text.rst index 47b678434fc899..92e7dd9a53b80d 100644 --- a/Doc/library/text.rst +++ b/Doc/library/text.rst @@ -16,6 +16,7 @@ Python's built-in string type in :ref:`textseq`. .. toctree:: string.rst + string.templatelib.rst re.rst difflib.rst textwrap.rst diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index e95fa3a6424e23..a416cbb4cc8eab 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -852,8 +852,8 @@ A literal pattern corresponds to most The rule ``strings`` and the token ``NUMBER`` are defined in the :doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` are -not supported. +supported. Raw strings and byte strings are supported. :ref:`f-strings` +and :ref:`t-strings` are not supported. The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are for expressing :ref:`complex numbers `; they require a real number diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 567c70111c20ec..a7f8e5392b7e71 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -561,9 +561,9 @@ escapes are not treated specially. single: f'; formatted string literal single: f"; formatted string literal -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw +A string literal with ``f`` or ``F`` in its prefix is a +:dfn:`formatted string literal`; see :ref:`f-strings`. The ``f`` may be +combined with ``r``, but not with ``b`` or ``u``, therefore raw formatted strings are possible, but formatted bytes literals are not. In triple-quoted literals, unescaped newlines and quotes are allowed (and are @@ -756,7 +756,7 @@ f-strings .. versionadded:: 3.6 A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain +that is prefixed with ``f`` or ``F``. These strings may contain replacement fields, which are expressions delimited by curly braces ``{}``. While other string literals always have a constant value, formatted strings are really expressions evaluated at run time. @@ -913,6 +913,48 @@ See also :pep:`498` for the proposal that added formatted string literals, and :meth:`str.format`, which uses a related format string mechanism. +.. _t-strings: +.. _template-string-literals: + +t-strings +--------- + +.. versionadded:: 3.14 + +A :dfn:`template string literal` or :dfn:`t-string` is a string literal +that is prefixed with ``t`` or ``T``. These strings follow the same +syntax and evaluation rules as :ref:`formatted string literals `, with +the following differences: + +- Rather than evaluating to a ``str`` object, t-strings evaluate to a + :class:`~string.templatelib.Template` object from the + :mod:`string.templatelib` module. + +- The :func:`format` protocol is not used. Instead, the format specifier and + conversions (if any) are passed to a new :class:`~string.templatelib.Interpolation` + object that is created for each evaluated expression. It is up to code that + processes the resulting :class:`~string.templatelib.Template` object to + decide how to handle format specifiers and conversions. + +- Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the expression ``{precision}`` before setting the ``format_spec`` + attribute of the resulting :class:`!Interpolation` object; if ``precision`` + is (for example) ``2``, the resulting format specifier will be ``'.2f'``. + +- When the equal sign ``'='`` is provided in an interpolation expression, the + resulting :class:`~string.templatelib.Template` object will have the expression + text along with a ``'='`` character placed in its + :attr:`~string.templatelib.Template.strings` attribute. The + :attr:`~string.templatelib.Template.interpolations` attribute will also + contain an ``Interpolation`` instance for the expression. By default, the + :attr:`~string.templatelib.Interpolation.conversion` attribute will be set to + ``'r'`` (that is, :func:`repr`), unless there is a conversion explicitly + specified (in which case it overrides the default) or a format specifier is + provided (in which case, the ``conversion`` defaults to ``None``). + + .. _numbers: Numeric literals diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 35b8c7cd8eb049..ea546c6a29df44 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,10 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. +The :mod:`string` module also contains support for so-called +:ref:`$-strings ` that offer yet another way to +substitute values into strings, using placeholders like ``$x`` and replacing +them with values from a dictionary. This syntax is easy to use, although +it offers much less control of the formatting. .. index:: single: formatted string literal From 4f1f648c3bd36a875501778786bb8d718ab33869 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 13:25:35 +0200 Subject: [PATCH 704/713] [3.14] gh-136251: Improvements to WASM demo REPL (GH-136252) (GH-136977) (cherry picked from commit d1d526afe7ce62c787b150652a2ba136cb949d74) Co-authored-by: adam j hartz Co-authored-by: Hood Chatham --- Makefile.pre.in | 4 +- ...-07-05-15-10-42.gh-issue-136251.GRM6o8.rst | 1 + Python/emscripten_syscalls.c | 2 +- Tools/wasm/README.md | 10 +- .../web_example/{python.html => index.html} | 362 +++++++++++++++--- configure | 2 +- configure.ac | 2 +- 7 files changed, 327 insertions(+), 56 deletions(-) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst rename Tools/wasm/emscripten/web_example/{python.html => index.html} (52%) diff --git a/Makefile.pre.in b/Makefile.pre.in index 66b34b779f27cb..fae5e384d3245f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1096,7 +1096,7 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten browser web example WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ -web_example/python.html: $(WEBEX_DIR)/python.html +web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ @@ -1124,7 +1124,7 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst b/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst new file mode 100644 index 00000000000000..6a35afe15e3875 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst @@ -0,0 +1 @@ +Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c index 886262acbc6810..d3eedad30e3639 100644 --- a/Python/emscripten_syscalls.c +++ b/Python/emscripten_syscalls.c @@ -100,7 +100,7 @@ EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { return; } const origResolveGlobalSymbol = resolveGlobalSymbol; - if (!Module.onExit && process?.exit) { + if (!Module.onExit && globalThis?.process?.exit) { Module.onExit = (code) => process.exit(code); } // * wrap the main symbol with WebAssembly.promising, diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 232321c515721e..9288598a0abc29 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -22,7 +22,7 @@ https://github.com/psf/webassembly for more information. ### Build To cross compile to the ``wasm32-emscripten`` platform you need -[the Emscripten compiler toolchain](https://emscripten.org/), +[the Emscripten compiler toolchain](https://emscripten.org/), a Python interpreter, and an installation of Node version 18 or newer. Emscripten version 4.0.2 is recommended; newer versions may also work, but all official testing is performed with that version. All commands below are relative @@ -86,11 +86,11 @@ CLI you will need to write your own alternative to `node_entry.mjs`. ### The Web Example -When building for Emscripten, the web example will be built automatically. It is -in the ``web_example`` directory. To run the web example, ``cd`` into the +When building for Emscripten, the web example will be built automatically. It +is in the ``web_example`` directory. To run the web example, ``cd`` into the ``web_example`` directory, then run ``python server.py``. This will start a web -server; you can then visit ``http://localhost:8000/python.html`` in a browser to -see a simple REPL example. +server; you can then visit ``http://localhost:8000/`` in a browser to see a +simple REPL example. The web example relies on a bug fix in Emscripten version 3.1.73 so if you build with earlier versions of Emscripten it may not work. The web example uses diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/index.html similarity index 52% rename from Tools/wasm/emscripten/web_example/python.html rename to Tools/wasm/emscripten/web_example/index.html index 078f86eb764419..9c89c9c0ed3bf5 100644 --- a/Tools/wasm/emscripten/web_example/python.html +++ b/Tools/wasm/emscripten/web_example/index.html @@ -1,31 +1,39 @@ - - - - - + + + + + wasm-python terminal + > + + +
+ + + diff --git a/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs new file mode 100644 index 00000000000000..5642372c9d2472 --- /dev/null +++ b/Tools/wasm/emscripten/web_example_pyrepl_jspi/src.mjs @@ -0,0 +1,194 @@ +// Much of this is adapted from here: +// https://github.com/mame/xterm-pty/blob/main/emscripten-pty.js +// Thanks to xterm-pty for making this possible! + +import createEmscriptenModule from "./python.mjs"; +import { openpty } from "https://unpkg.com/xterm-pty/index.mjs"; +import "https://unpkg.com/@xterm/xterm/lib/xterm.js"; + +var term = new Terminal(); +term.open(document.getElementById("terminal")); +const { master, slave: PTY } = openpty(); +term.loadAddon(master); +globalThis.PTY = PTY; + +async function setupStdlib(Module) { + const versionInt = Module.HEAPU32[Module._Py_Version >>> 2]; + const major = (versionInt >>> 24) & 0xff; + const minor = (versionInt >>> 16) & 0xff; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); +} + +const tty_ops = { + ioctl_tcgets: () => { + const termios = PTY.ioctl("TCGETS"); + const data = { + c_iflag: termios.iflag, + c_oflag: termios.oflag, + c_cflag: termios.cflag, + c_lflag: termios.lflag, + c_cc: termios.cc, + }; + return data; + }, + + ioctl_tcsets: (_tty, _optional_actions, data) => { + PTY.ioctl("TCSETS", { + iflag: data.c_iflag, + oflag: data.c_oflag, + cflag: data.c_cflag, + lflag: data.c_lflag, + cc: data.c_cc, + }); + return 0; + }, + + ioctl_tiocgwinsz: () => PTY.ioctl("TIOCGWINSZ").reverse(), + + get_char: () => { + throw new Error("Should not happen"); + }, + put_char: () => { + throw new Error("Should not happen"); + }, + + fsync: () => {}, +}; + +const POLLIN = 1; +const POLLOUT = 4; + +const waitResult = { + READY: 0, + SIGNAL: 1, + TIMEOUT: 2, +}; + +function onReadable() { + var handle; + var promise = new Promise((resolve) => { + handle = PTY.onReadable(() => resolve(waitResult.READY)); + }); + return [promise, handle]; +} + +function onSignal() { + // TODO: signal handling + var handle = { dispose() {} }; + var promise = new Promise((resolve) => {}); + return [promise, handle]; +} + +function onTimeout(timeout) { + var id; + var promise = new Promise((resolve) => { + if (timeout > 0) { + id = setTimeout(resolve, timeout, waitResult.TIMEOUT); + } + }); + var handle = { + dispose() { + if (id) { + clearTimeout(id); + } + }, + }; + return [promise, handle]; +} + +async function waitForReadable(timeout) { + let p1, p2, p3; + let h1, h2, h3; + try { + [p1, h1] = onReadable(); + [p2, h2] = onTimeout(timeout); + [p3, h3] = onSignal(); + return await Promise.race([p1, p2, p3]); + } finally { + h1.dispose(); + h2.dispose(); + h3.dispose(); + } +} + +const FIONREAD = 0x541b; + +const tty_stream_ops = { + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + let readBytes = PTY.read(length); + if (length && !readBytes.length) { + const status = await waitForReadable(-1); + if (status === waitResult.READY) { + readBytes = PTY.read(length); + } else { + throw new Error("Not implemented"); + } + } + buffer.set(readBytes, offset); + return readBytes.length; + }, + + write: (stream, buffer, offset, length) => { + // Note: default `buffer` is for some reason `HEAP8` (signed), while we want unsigned `HEAPU8`. + buffer = new Uint8Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength, + ); + const toWrite = Array.from(buffer.subarray(offset, offset + length)); + PTY.write(toWrite); + return length; + }, + + async pollAsync(stream, timeout) { + if (!PTY.readable && timeout) { + await waitForReadable(timeout); + } + return (PTY.readable ? POLLIN : 0) | (PTY.writable ? POLLOUT : 0); + }, + ioctl(stream, request, varargs) { + if (request === FIONREAD) { + const res = PTY.fromLdiscToUpperBuffer.length; + Module.HEAPU32[varargs / 4] = res; + return 0; + } + throw new Error("Unimplemented ioctl request"); + }, +}; + +async function setupStdio(Module) { + Object.assign(Module.TTY.default_tty_ops, tty_ops); + Object.assign(Module.TTY.stream_ops, tty_stream_ops); +} + +const emscriptenSettings = { + async preRun(Module) { + Module.addRunDependency("pre-run"); + Module.ENV.TERM = "xterm-256color"; + // Uncomment next line to turn on tracing (messages go to browser console). + // Module.ENV.PYREPL_TRACE = "1"; + + // Leak module so we can try to show traceback if we crash on startup + globalThis.Module = Module; + await Promise.all([setupStdlib(Module), setupStdio(Module)]); + Module.removeRunDependency("pre-run"); + }, +}; + +try { + await createEmscriptenModule(emscriptenSettings); +} catch (e) { + // Show JavaScript exception and traceback + console.warn(e); + // Show Python exception and traceback + Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + process.exit(1); +} diff --git a/configure b/configure index 946cd471f64d32..345822a6c6df76 100755 --- a/configure +++ b/configure @@ -9603,7 +9603,7 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY" as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" as_fn_append LINKFORSHARED " -sTEXTDECODER=2" @@ -31196,9 +31196,7 @@ case $ac_sys_system in #( - py_cv_module_fcntl=n/a py_cv_module_readline=n/a - py_cv_module_termios=n/a py_cv_module_=n/a ;; #( diff --git a/configure.ac b/configure.ac index 6b15beb050c1af..d6059471771871 100644 --- a/configure.ac +++ b/configure.ac @@ -2335,7 +2335,7 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY"]) AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) dnl Avoid bugs in JS fallback string decoding path @@ -7778,9 +7778,7 @@ AS_CASE([$ac_sys_system], ) dnl fcntl, readline, and termios are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( - [fcntl], [readline], - [termios], ) ], [WASI], [ From c328d140eb85df760a40ca58ab0b27195c550560 Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 18:15:27 +0200 Subject: [PATCH 708/713] [3.14] Fix code example in `annotationlib` documentation (GH-136972) (#137002) Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> --- Doc/library/annotationlib.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index 7dfc11449a6cbc..981d89be7d58d6 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -511,7 +511,7 @@ code execution even with no access to any globals or builtins. For example: >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass ... - >>> annotationlib.get_annotations(f, format=annotationlib.Format.SOURCE) + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) Hello world {'x': 'None'} From 805daa2edb19b78f76291d7a3147a06a5670a112 Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 22 Jul 2025 19:16:31 +0300 Subject: [PATCH 709/713] [3.14] Revert "gh-112068: C API: Add support of nullable arguments in PyArg_Parse (GH-121303)" (GH-136991) (#137006) --- Doc/c-api/arg.rst | 15 -- Doc/whatsnew/3.14.rst | 5 - Lib/test/test_capi/test_getargs.py | 117 --------- Lib/test/test_mmap.py | 2 +- Misc/NEWS.d/3.14.0b1.rst | 2 +- ...-07-22-15-18-08.gh-issue-112068.4WvT-8.rst | 1 + Modules/_ctypes/_ctypes.c | 12 +- Modules/_interpretersmodule.c | 9 +- Modules/_json.c | 13 +- Modules/_threadmodule.c | 18 +- Modules/mmapmodule.c | 15 +- Python/getargs.c | 247 ++++++------------ 12 files changed, 139 insertions(+), 317 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index ab9f9c4539ae9a..112169635d999a 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` - structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -387,17 +383,6 @@ Other objects Non-tuple sequences are deprecated if *items* contains format units which store a borrowed buffer or a borrowed reference. -``unit?`` (anything or ``None``) [*matching-variable(s)*] - ``?`` modifies the behavior of the preceding format unit. - The C variable(s) corresponding to that parameter should be initialized - to their default value --- when the argument is ``None``, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding - C variable(s). - If the argument is not ``None``, it is parsed according to the specified - format unit. - - .. versionadded:: 3.14 - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 2906828e05d229..2517dd93885d7f 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -2940,11 +2940,6 @@ New features file. (Contributed by Victor Stinner in :gh:`127350`.) -* Add support of nullable arguments in :c:func:`PyArg_ParseTuple` and - similar functions. - Adding ``?`` after any format unit makes ``None`` be accepted as a value. - (Contributed by Serhiy Storchaka in :gh:`112068`.) - * The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and similar functions now use :meth:`~object.__index__` if available, like all other integer formats. diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 67a8da7599511f..700870b6266717 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -1389,123 +1389,6 @@ def test_nested_sequence(self): "argument 1 must be sequence of length 1, not 0"): parse(([],), {}, '(' + f + ')', ['a']) - def test_specific_type_errors(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, expected, got='list'): - errmsg = f'must be {expected}, not {got}' - with self.assertRaisesRegex(TypeError, errmsg): - parse((arg,), {}, format, ['a']) - - check('k', [], 'int') - check('k?', [], 'int or None') - check('K', [], 'int') - check('K?', [], 'int or None') - check('c', [], 'a byte string of length 1') - check('c?', [], 'a byte string of length 1 or None') - check('c', b'abc', 'a byte string of length 1', - 'a bytes object of length 3') - check('c?', b'abc', 'a byte string of length 1 or None', - 'a bytes object of length 3') - check('c', bytearray(b'abc'), 'a byte string of length 1', - 'a bytearray object of length 3') - check('c?', bytearray(b'abc'), 'a byte string of length 1 or None', - 'a bytearray object of length 3') - check('C', [], 'a unicode character') - check('C?', [], 'a unicode character or None') - check('C', 'abc', 'a unicode character', - 'a string of length 3') - check('C?', 'abc', 'a unicode character or None', - 'a string of length 3') - check('s', [], 'str') - check('s?', [], 'str or None') - check('z', [], 'str or None') - check('z?', [], 'str or None') - check('es', [], 'str') - check('es?', [], 'str or None') - check('es#', [], 'str') - check('es#?', [], 'str or None') - check('et', [], 'str, bytes or bytearray') - check('et?', [], 'str, bytes, bytearray or None') - check('et#', [], 'str, bytes or bytearray') - check('et#?', [], 'str, bytes, bytearray or None') - check('w*', [], 'read-write bytes-like object') - check('w*?', [], 'read-write bytes-like object or None') - check('S', [], 'bytes') - check('S?', [], 'bytes or None') - check('U', [], 'str') - check('U?', [], 'str or None') - check('Y', [], 'bytearray') - check('Y?', [], 'bytearray or None') - check('(OO)', 42, '2-item tuple', 'int') - check('(OO)?', 42, '2-item tuple or None', 'int') - check('(OO)', (1, 2, 3), 'tuple of length 2', '3') - - def test_nullable(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, allows_none=False): - # Because some format units (such as y*) require cleanup, - # we force the parsing code to perform the cleanup by adding - # an argument that always fails. - # By checking for an exception, we ensure that the parsing - # of the first argument was successful. - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (None, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + 'b', ['a', 'b']) - self.assertRaises(OverflowError if allows_none else TypeError, parse, - (None, 256), {}, format + 'b', ['a', 'b']) - - check('b', 42) - check('B', 42) - check('h', 42) - check('H', 42) - check('i', 42) - check('I', 42) - check('n', 42) - check('l', 42) - check('k', 42) - check('L', 42) - check('K', 42) - check('f', 2.5) - check('d', 2.5) - check('D', 2.5j) - check('c', b'a') - check('C', 'a') - check('p', True, allows_none=True) - check('y', b'buffer') - check('y*', b'buffer') - check('y#', b'buffer') - check('s', 'string') - check('s*', 'string') - check('s#', 'string') - check('z', 'string', allows_none=True) - check('z*', 'string', allows_none=True) - check('z#', 'string', allows_none=True) - check('w*', bytearray(b'buffer')) - check('U', 'string') - check('S', b'bytes') - check('Y', bytearray(b'bytearray')) - check('O', object, allows_none=True) - - check('(OO)', (1, 2)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)?O)', ['a']), (1, 2, 3)) - self.assertEqual(parse(((None, 3),), {}, '((OO)?O)', ['a']), (NULL, NULL, 3)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)O)', ['a']), (1, 2, 3)) - self.assertRaises(TypeError, parse, ((None, 3),), {}, '((OO)O)', ['a']) - - parse((None,), {}, 'es?', ['a']) - parse((None,), {}, 'es#?', ['a']) - parse((None,), {}, 'et?', ['a']) - parse((None,), {}, 'et#?', ['a']) - parse((None,), {}, 'O!?', ['a']) - parse((None,), {}, 'O&?', ['a']) - - # TODO: More tests for es?, es#?, et?, et#?, O!, O& - @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') def test_gh_119213(self): rc, out, err = script_helper.assert_python_ok("-c", """if True: diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index fd4197b7086976..b2a299ed172967 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -732,7 +732,7 @@ def test_tagname(self): m2.close() m1.close() - with self.assertRaisesRegex(TypeError, 'must be str or None'): + with self.assertRaisesRegex(TypeError, 'tagname'): mmap.mmap(-1, 8, tagname=1) @cpython_only diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index f76896954f321d..f2dd631358550b 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -2012,7 +2012,7 @@ interpreter. .. nonce: ofI5Fl .. section: C API -Add support of nullable arguments in :c:func:`PyArg_Parse` and similar +[Reverted in :gh:`136991`] Add support of nullable arguments in :c:func:`PyArg_Parse` and similar functions. Adding ``?`` after any format unit makes ``None`` be accepted as a value. diff --git a/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst new file mode 100644 index 00000000000000..018c5c7880c001 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst @@ -0,0 +1 @@ +Revert support of nullable arguments in :c:func:`PyArg_Parse`. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 55a446d02b27f3..08e00adfd0c3e6 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3918,7 +3918,9 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) PyObject *name = Py_None; PyObject *defval; PyObject *typ; - if (!PyArg_ParseTuple(item, "i|U?O", &flag, &name, &defval)) { + if (!PyArg_ParseTuple(item, "i|OO", &flag, &name, &defval) || + !(name == Py_None || PyUnicode_Check(name))) + { PyErr_SetString(PyExc_TypeError, "paramflags must be a sequence of (int [,string [,value]]) tuples"); return 0; @@ -3983,8 +3985,10 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) void *handle; PyObject *paramflags = NULL; - if (!PyArg_ParseTuple(args, "O|O?", &ftuple, ¶mflags)) + if (!PyArg_ParseTuple(args, "O|O", &ftuple, ¶mflags)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ftuple = PySequence_Tuple(ftuple); if (!ftuple) @@ -4116,8 +4120,10 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) GUID *iid = NULL; Py_ssize_t iid_len = 0; - if (!PyArg_ParseTuple(args, "is|O?z#", &index, &name, ¶mflags, &iid, &iid_len)) + if (!PyArg_ParseTuple(args, "is|Oz#", &index, &name, ¶mflags, &iid, &iid_len)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 9426ce72733c28..faf3b25b68c4eb 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -1415,11 +1415,14 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) PyObject *idobj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O?|$p:get_config", kwlist, + "O|$p:get_config", kwlist, &idobj, &restricted)) { return NULL; } + if (idobj == Py_None) { + idobj = NULL; + } int reqready = 0; PyInterpreterState *interp = \ @@ -1536,14 +1539,14 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"exc", NULL}; PyObject *exc_arg = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O?:capture_exception", kwlist, + "|O:capture_exception", kwlist, &exc_arg)) { return NULL; } PyObject *exc = exc_arg; - if (exc == NULL) { + if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); if (exc == NULL) { Py_RETURN_NONE; diff --git a/Modules/_json.c b/Modules/_json.c index 6c381fab449db2..2f82a69a1579dc 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -1228,16 +1228,23 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; - PyObject *markers = Py_None, *defaultfn, *encoder, *indent, *key_separator; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!?OOOUUppp:make_encoder", kwlist, - &PyDict_Type, &markers, &defaultfn, &encoder, &indent, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) return NULL; + if (markers != Py_None && !PyDict_Check(markers)) { + PyErr_Format(PyExc_TypeError, + "make_encoder() argument 1 must be dict or None, " + "not %.200s", Py_TYPE(markers)->tp_name); + return NULL; + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 121b0f803715af..775cd704cda4ed 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -667,12 +667,12 @@ PyThreadHandleObject_join(PyObject *op, PyObject *args) PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); PyObject *timeout_obj = NULL; - if (!PyArg_ParseTuple(args, "|O?:join", &timeout_obj)) { + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } PyTime_t timeout_ns = -1; - if (timeout_obj != NULL) { + if (timeout_obj != NULL && timeout_obj != Py_None) { if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -1945,10 +1945,10 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, PyObject *func = NULL; int daemon = 1; thread_module_state *state = get_thread_state(module); - PyObject *hobj = Py_None; + PyObject *hobj = NULL; if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, - "O|O!?p:start_joinable_thread", keywords, - &func, state->thread_handle_type, &hobj, &daemon)) { + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { return NULL; } @@ -1958,6 +1958,14 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, return NULL; } + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, hobj) < 0) { return NULL; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index d9e8c60209dae3..e2ae967ce0c33d 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,6 +23,7 @@ #endif #include +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() @@ -515,7 +516,7 @@ mmap_read_method(PyObject *op, PyObject *args) mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|n?:read", &num_bytes)) + if (!PyArg_ParseTuple(args, "|O&:read", _Py_convert_optional_to_ssize_t, &num_bytes)) return NULL; CHECK_VALID(NULL); @@ -1709,7 +1710,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - PyObject *tagname = NULL; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1719,7 +1720,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|U?iL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1852,7 +1853,13 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL) { + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); diff --git a/Python/getargs.c b/Python/getargs.c index 0cf596285cc7b5..9d3dea0cb4364b 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,8 +1,6 @@ /* New getargs implementation */ -#include - #define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() @@ -468,12 +466,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; - bool nullable = false; int istuple = PyTuple_Check(arg); int mustbetuple = istuple; - assert(*format == '('); - format++; for (;;) { int c = *format++; if (c == '(') { @@ -482,12 +477,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, level++; } else if (c == ')') { - if (level == 0) { - if (*format == '?') { - nullable = true; - } + if (level == 0) break; - } level--; } else if (c == ':' || c == ';' || c == '\0') @@ -524,13 +515,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - if (arg == Py_None && nullable) { - const char *msg = skipitem(p_format, p_va, flags); - if (msg != NULL) { - levels[0] = 0; - } - return msg; - } if (istuple) { /* fallthrough */ } @@ -539,10 +523,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item tuple%s, not %.50s", - n, - nullable ? " or None" : "", - arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); + "must be %d-item tuple, not %.50s", + n, + arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } else { @@ -579,7 +562,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return msgbuf; } - format = *p_format + 1; + format = *p_format; for (i = 0; i < n; i++) { const char *msg; PyObject *item = PyTuple_GET_ITEM(arg, i); @@ -594,10 +577,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - format++; - if (*format == '?') { - format++; - } *p_format = format; if (!istuple) { Py_DECREF(arg); @@ -616,8 +595,11 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; if (*format == '(' /* ')' */) { + format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, bufsize, freelist); + if (msg == NULL) + format++; } else { msg = convertsimple(arg, &format, p_va, flags, @@ -647,7 +629,7 @@ _PyArg_BadArgument(const char *fname, const char *displayname, } static const char * -converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); assert(arg != NULL); @@ -657,23 +639,20 @@ converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, siz } else { PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s", expected, - nullable ? " or None" : "", + "must be %.50s, not %.50s", expected, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); } return msgbuf; } static const char * -convertcharerr(bool nullable, const char *expected, const char *what, Py_ssize_t size, +convertcharerr(const char *expected, const char *what, Py_ssize_t size, char *msgbuf, size_t bufsize) { assert(expected != NULL); PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s of length %zd", - expected, - nullable ? " or None" : "", - what, size); + "must be %.50s, not %.50s of length %zd", + expected, what, size); return msgbuf; } @@ -693,26 +672,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { #define RETURN_ERR_OCCURRED return msgbuf -#define HANDLE_NULLABLE \ - if (*format == '?') { \ - format++; \ - if (arg == Py_None) { \ - break; \ - } \ - nullable = true; \ - } - const char *format = *p_format; char c = *format++; const char *sarg; - bool nullable = false; switch (c) { case 'b': { /* unsigned byte -- very short int */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -726,6 +694,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "unsigned byte integer is greater than maximum"); RETURN_ERR_OCCURRED; } + else *p = (unsigned char) ival; break; } @@ -733,7 +702,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -744,7 +712,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'h': {/* signed short int */ short *p = va_arg(*p_va, short *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -766,7 +733,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ unsigned short *p = va_arg(*p_va, unsigned short *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -777,7 +743,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -799,7 +764,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'I': { /* int sized bitfield, both signed and unsigned allowed */ unsigned int *p = va_arg(*p_va, unsigned int *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -812,7 +776,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - HANDLE_NULLABLE; Py_ssize_t ival = -1; iobj = _PyNumber_Index(arg); if (iobj != NULL) { @@ -826,7 +789,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'l': {/* long int */ long *p = va_arg(*p_va, long *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -837,10 +799,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); - HANDLE_NULLABLE; unsigned long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)(long)-1 && PyErr_Occurred()) { @@ -852,7 +813,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'L': {/* long long */ long long *p = va_arg( *p_va, long long * ); - HANDLE_NULLABLE; long long ival = PyLong_AsLongLong(arg); if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -863,10 +823,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'K': { /* long long sized bitfield */ unsigned long long *p = va_arg(*p_va, unsigned long long *); - HANDLE_NULLABLE; unsigned long long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } ival = PyLong_AsUnsignedLongLongMask(arg); if (ival == (unsigned long long)(long long)-1 && PyErr_Occurred()) { @@ -878,7 +837,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'f': {/* float */ float *p = va_arg(*p_va, float *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -889,7 +847,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'd': {/* double */ double *p = va_arg(*p_va, double *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -900,7 +857,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); - HANDLE_NULLABLE; Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) @@ -912,10 +868,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) { if (PyBytes_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytes object", PyBytes_GET_SIZE(arg), msgbuf, bufsize); } @@ -923,28 +878,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (PyByteArray_Check(arg)) { if (PyByteArray_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytearray object", PyByteArray_GET_SIZE(arg), msgbuf, bufsize); } *p = PyByteArray_AS_STRING(arg)[0]; } else - return converterr(nullable, "a byte string of length 1", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int kind; const void *data; if (!PyUnicode_Check(arg)) - return converterr(nullable, "a unicode character", arg, msgbuf, bufsize); + return converterr("a unicode character", arg, msgbuf, bufsize); if (PyUnicode_GET_LENGTH(arg) != 1) { - return convertcharerr(nullable, "a unicode character", + return convertcharerr("a unicode character", "a string", PyUnicode_GET_LENGTH(arg), msgbuf, bufsize); } @@ -957,7 +911,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'p': {/* boolean *p*redicate */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int val = PyObject_IsTrue(arg); if (val > 0) *p = 1; @@ -976,31 +929,24 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count; if (*format == '*') { - format++; - HANDLE_NULLABLE; if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); + format++; if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } - else if (*format == '#') { + count = convertbuffer(arg, (const void **)p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); *psize = count; - } - else { - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + format++; + } else { if (strlen(*p) != (size_t)count) { PyErr_SetString(PyExc_ValueError, "embedded null byte"); RETURN_ERR_OCCURRED; @@ -1016,35 +962,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, (void *)sarg, len, 1, 0); } else { /* any bytes-like object */ const char *buf; if (getbuffer(arg, p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); } if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } + format++; } else if (*format == '#') { /* a string or read-only bytes-like object */ /* "s#" or "z#" */ const void **p = (const void **)va_arg(*p_va, const char **); Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) { *p = NULL; *psize = 0; @@ -1053,7 +996,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); *p = sarg; *psize = len; @@ -1063,22 +1006,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); *psize = count; } + format++; } else { /* "s" or "z" */ const char **p = va_arg(*p_va, const char **); Py_ssize_t len; sarg = NULL; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); if (strlen(sarg) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "embedded null character"); @@ -1087,7 +1030,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = sarg; } else - return converterr(c == 'z' || nullable, "str", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); } break; @@ -1116,46 +1059,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, recode_strings = 0; else return converterr( - nullable, "(unknown parser marker combination)", + "(unknown parser marker combination)", arg, msgbuf, bufsize); buffer = (char **)va_arg(*p_va, char **); format++; if (buffer == NULL) - return converterr(nullable, "(buffer is NULL)", + return converterr("(buffer is NULL)", arg, msgbuf, bufsize); - Py_ssize_t *psize = NULL; - if (*format == '#') { - /* Using buffer length parameter '#': - - - if *buffer is NULL, a new buffer of the - needed size is allocated and the data - copied into it; *buffer is updated to point - to the new buffer; the caller is - responsible for PyMem_Free()ing it after - usage - - - if *buffer is not NULL, the data is - copied to *buffer; *buffer_len has to be - set to the size of the buffer on input; - buffer overflow is signalled with an error; - buffer has to provide enough room for the - encoded string plus the trailing 0-byte - - - in both cases, *buffer_len is updated to - the size of the buffer /excluding/ the - trailing 0-byte - - */ - psize = va_arg(*p_va, Py_ssize_t*); - - format++; - if (psize == NULL) { - return converterr( - nullable, "(buffer_len is NULL)", - arg, msgbuf, bufsize); - } - } - HANDLE_NULLABLE; /* Encode object */ if (!recode_strings && @@ -1176,7 +1086,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, encoding, NULL); if (s == NULL) - return converterr(nullable, "(encoding failed)", + return converterr("(encoding failed)", arg, msgbuf, bufsize); assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); @@ -1186,15 +1096,42 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { return converterr( - nullable, - recode_strings ? "str" - : nullable ? "str, bytes, bytearray" - : "str, bytes or bytearray", + recode_strings ? "str" : "str, bytes or bytearray", arg, msgbuf, bufsize); } /* Write output; output is guaranteed to be 0-terminated */ - if (psize != NULL) { + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer of the + needed size is allocated and the data + copied into it; *buffer is updated to point + to the new buffer; the caller is + responsible for PyMem_Free()ing it after + usage + + - if *buffer is not NULL, the data is + copied to *buffer; *buffer_len has to be + set to the size of the buffer on input; + buffer overflow is signalled with an error; + buffer has to provide enough room for the + encoded string plus the trailing 0-byte + + - in both cases, *buffer_len is updated to + the size of the buffer /excluding/ the + trailing 0-byte + + */ + Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); + + format++; + if (psize == NULL) { + Py_DECREF(s); + return converterr( + "(buffer_len is NULL)", + arg, msgbuf, bufsize); + } if (*buffer == NULL) { *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { @@ -1205,7 +1142,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } } else { @@ -1239,7 +1176,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( - nullable, "encoded string without null bytes", + "encoded string without null bytes", arg, msgbuf, bufsize); } *buffer = PyMem_NEW(char, size + 1); @@ -1250,7 +1187,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } memcpy(*buffer, ptr, size+1); @@ -1261,32 +1198,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'S': { /* PyBytes object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) *p = arg; else - return converterr(nullable, "bytes", arg, msgbuf, bufsize); + return converterr("bytes", arg, msgbuf, bufsize); break; } case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyByteArray_Check(arg)) *p = arg; else - return converterr(nullable, "bytearray", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyUnicode_Check(arg)) { *p = arg; } else - return converterr(nullable, "str", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } @@ -1297,11 +1231,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, type = va_arg(*p_va, PyTypeObject*); p = va_arg(*p_va, PyObject **); format++; - HANDLE_NULLABLE; if (PyType_IsSubtype(Py_TYPE(arg), type)) *p = arg; else - return converterr(nullable, type->tp_name, arg, msgbuf, bufsize); + return converterr(type->tp_name, arg, msgbuf, bufsize); } else if (*format == '&') { @@ -1310,18 +1243,16 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void *addr = va_arg(*p_va, void *); int res; format++; - HANDLE_NULLABLE; if (! (res = (*convert)(arg, addr))) - return converterr(nullable, "(unspecified)", + return converterr("(unspecified)", arg, msgbuf, bufsize); if (res == Py_CLEANUP_SUPPORTED && addcleanup(addr, freelist, convert) == -1) - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; *p = arg; } break; @@ -1333,30 +1264,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format != '*') return converterr( - nullable, "(invalid use of 'w' format character)", + "(invalid use of 'w' format character)", arg, msgbuf, bufsize); format++; - HANDLE_NULLABLE; /* Caller is interested in Py_buffer, and the object supports it directly. The request implicitly asks for PyBUF_SIMPLE, so the result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - return converterr(nullable, "read-write bytes-like object", + return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } default: - return converterr(nullable, "(impossible)", arg, msgbuf, bufsize); + return converterr("(impossible)", arg, msgbuf, bufsize); } @@ -2751,9 +2681,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) return "impossible"; } - if (*format == '?') { - format++; - } *p_format = format; return NULL; From 715c658bf87d00874a0c1a1db10ea3f9b29ab48b Mon Sep 17 00:00:00 2001 From: "Miss Islington (bot)" <31488909+miss-islington@users.noreply.github.com> Date: Tue, 22 Jul 2025 18:21:19 +0200 Subject: [PATCH 710/713] [3.14] Fix tables in 'Using on Windows' for the text writer (GH-137012) (#137015) Co-authored-by: Adam Turner <9087854+AA-Turner@users.noreply.github.com> --- Doc/using/windows.rst | 362 +++++++++++++++++++++++++----------------- 1 file changed, 213 insertions(+), 149 deletions(-) diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 9628da3d2f6b12..a5e203a867724f 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -385,50 +385,77 @@ The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 - - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 + +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - ``PYTHON_MANAGER_DEFAULT`` + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - ``PYTHON_MANAGER_AUTOMATIC_INSTALL`` + - True to allow automatic installs when specifying a particular runtime + to launch. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -492,7 +519,7 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no @@ -638,46 +665,64 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 - - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 + +.. rubric:: Administrative configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Description + + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. .. _install-freethreaded-windows: @@ -716,70 +761,89 @@ issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 - - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " - "``pip`` gives me a ""command not found"" error when I type it in my - terminal.","Have you activated a virtual environment? Run the - ``.venv\Scripts\activate`` script in your terminal to activate. - " - "","The package may be available but missing the generated executable. - We recommend using the ``python -m pip`` command instead, or alternatively - the ``python -m pip install --force pip`` command will recreate the - executables and show you the path to add to :envvar:`PATH`. These scripts are - separated for each runtime, and so you may need to add multiple paths. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 + +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your ``PYTHON_MANAGER_DEFAULT`` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead, + or alternatively the ``python -m pip install --force pip`` command + will recreate the executables and show you the path to + add to :envvar:`PATH`. + These scripts are separated for each runtime, and so you may need to + add multiple paths. .. _windows-embeddable: From 70990ed5cad5b886a7f595923724706a222c5945 Mon Sep 17 00:00:00 2001 From: Jelle Zijlstra Date: Tue, 22 Jul 2025 09:23:53 -0700 Subject: [PATCH 711/713] =?UTF-8?q?[3.14]=20Revert=20"[3.14]=20gh-135228:?= =?UTF-8?q?=20When=20@dataclass(slots=3DTrue)=20replaces=E2=80=A6=20(#1370?= =?UTF-8?q?13)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Lib/dataclasses.py | 15 -------- Lib/test/test_dataclasses/__init__.py | 35 ------------------- ...-07-20-16-56-55.gh-issue-135228.n_XIao.rst | 4 --- 3 files changed, 54 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index 22b78bb2fbe6ed..83ea623dce6281 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -1338,13 +1338,6 @@ def _add_slots(cls, is_frozen, weakref_slot, defined_fields): or _update_func_cell_for__class__(member.fdel, cls, newcls)): break - # gh-135228: Make sure the original class can be garbage collected. - # Bypass mapping proxy to allow __dict__ to be removed - old_cls_dict = cls.__dict__ | _deproxier - old_cls_dict.pop('__dict__', None) - if "__weakref__" in cls.__dict__: - del cls.__weakref__ - return newcls @@ -1739,11 +1732,3 @@ def _replace(self, /, **changes): # changes that aren't fields, this will correctly raise a # TypeError. return self.__class__(**changes) - - -# Hack to the get the underlying dict out of a mappingproxy -# Use it with: cls.__dict__ | _deproxier -class _Deproxier: - def __ror__(self, other): - return other -_deproxier = _Deproxier() diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index 6bf5e5b3e5554b..e98a8f284cec9f 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -3804,41 +3804,6 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) - def test_original_class_is_gced(self): - # gh-135228: Make sure when we replace the class with slots=True, the original class - # gets garbage collected. - def make_simple(): - @dataclass(slots=True) - class SlotsTest: - pass - - return SlotsTest - - def make_with_annotations(): - @dataclass(slots=True) - class SlotsTest: - x: int - - return SlotsTest - - def make_with_annotations_and_method(): - @dataclass(slots=True) - class SlotsTest: - x: int - - def method(self) -> int: - return self.x - - return SlotsTest - - for make in (make_simple, make_with_annotations, make_with_annotations_and_method): - with self.subTest(make=make): - C = make() - support.gc_collect() - candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' - and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] - self.assertEqual(candidates, [C]) - class TestDescriptors(unittest.TestCase): def test_set_name(self): diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst deleted file mode 100644 index ee8962c6f46e75..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-20-16-56-55.gh-issue-135228.n_XIao.rst +++ /dev/null @@ -1,4 +0,0 @@ -When :mod:`dataclasses` replaces a class with a slotted dataclass, the -original class is now garbage collected again. Earlier changes in Python -3.14 caused this class to remain in existence together with the replacement -class synthesized by :mod:`dataclasses`. From 48f88310044c6ef877f3b0761cf7afece2f8fb3a Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> Date: Tue, 22 Jul 2025 19:42:27 +0300 Subject: [PATCH 712/713] Python 3.14.0rc1 --- Include/patchlevel.h | 6 +- Lib/pydoc_data/topics.py | 33 +- Misc/NEWS.d/3.14.0rc1.rst | 316 ++++++++++++++++++ ...-07-18-17-15-00.gh-issue-135621.9cyCNb.rst | 2 - ...-05-20-17-13-51.gh-issue-134009.CpCmry.rst | 1 - ...-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst | 3 - ...-07-22-15-18-08.gh-issue-112068.4WvT-8.rst | 1 - ...-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst | 1 - ...5-05-17-20-56-05.gh-issue-91153.afgtG2.rst | 1 - ...-06-03-21-06-22.gh-issue-133136.Usnvri.rst | 2 - ...-06-06-02-24-42.gh-issue-135148.r-t2sC.rst | 3 - ...-06-12-00-03-34.gh-issue-116738.iBBAdo.rst | 1 - ...-07-08-23-22-08.gh-issue-132661.34ftJl.rst | 5 - ...-07-08-23-53-51.gh-issue-132661.B84iYt.rst | 1 - ...-07-10-15-53-16.gh-issue-136525.xAko0e.rst | 2 - ...-07-10-23-23-50.gh-issue-136517._NHJyv.rst | 2 - ...-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst | 3 - ...-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst | 1 - ...-07-19-12-37-05.gh-issue-136801.XU_tF2.rst | 1 - ...-05-25-11-02-05.gh-issue-134657.3YFhR9.rst | 1 - ...-06-28-11-32-57.gh-issue-134759.AjjKcG.rst | 2 - ...-07-07-22-12-32.gh-issue-136380.1b_nXl.rst | 3 - ...-07-08-20-58-01.gh-issue-136434.uuJsjS.rst | 2 - ...-07-09-20-29-30.gh-issue-136476.HyLLzh.rst | 2 - ...-07-10-00-47-37.gh-issue-136470.KlUEUG.rst | 2 - ...5-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst | 3 - ...-07-11-03-39-15.gh-issue-136523.s7caKL.rst | 1 - ...-07-11-23-04-39.gh-issue-136549.oAi8u4.rst | 1 - ...-07-19-16-20-54.gh-issue-130645.O-dYcN.rst | 1 - ...-07-20-16-02-00.gh-issue-136874.cLC3o1.rst | 1 - ...-07-21-16-10-24.gh-issue-124621.wyoWc1.rst | 1 - ...-07-21-22-35-50.gh-issue-136170.QUlc78.rst | 3 - ...-06-09-20-38-25.gh-issue-118350.KgWCcP.rst | 2 - ...-07-21-14-15-25.gh-issue-135661.nAxXw5.rst | 2 - ...-07-05-15-10-42.gh-issue-136251.GRM6o8.rst | 1 - README.rst | 4 +- 36 files changed, 340 insertions(+), 77 deletions(-) create mode 100644 Misc/NEWS.d/3.14.0rc1.rst delete mode 100644 Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst delete mode 100644 Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst delete mode 100644 Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst delete mode 100644 Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst delete mode 100644 Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst delete mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 77fc8aeb918cf6..c926ee8c32b738 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,11 +20,11 @@ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 14 #define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 4 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_GAMMA +#define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.14.0b4+" +#define PY_VERSION "3.14.0rc1" /*--end constants--*/ diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index e32517960d44d1..71b1917de1c395 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue Jul 8 11:57:16 2025 +# Autogenerated by Sphinx on Tue Jul 22 19:42:37 2025 # as part of the release process. topics = { @@ -2328,7 +2328,8 @@ def foo(): The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and -byte strings are supported. f-strings are not supported. +byte strings are supported. f-strings and t-strings are not +supported. The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" are for expressing complex numbers; they require a real number on the @@ -5232,9 +5233,9 @@ class of the instance or a *non-virtual base class* thereof. The The "str.format()" method and the "Formatter" class share the same syntax for format strings (although in the case of "Formatter", subclasses can define their own format string syntax). The syntax is -related to that of formatted string literals, but it is less -sophisticated and, in particular, does not support arbitrary -expressions. +related to that of formatted string literals and template string +literals, but it is less sophisticated and, in particular, does not +support arbitrary expressions. Format strings contain “replacement fields” surrounded by curly braces "{}". Anything that is not contained in braces is considered literal @@ -5334,9 +5335,9 @@ class of the instance or a *non-virtual base class* thereof. The “Format specifications” are used within replacement fields contained within a format string to define how individual values are presented -(see Format String Syntax and f-strings). They can also be passed -directly to the built-in "format()" function. Each formattable type -may define how the format specification is to be interpreted. +(see Format String Syntax, f-strings, and t-strings). They can also be +passed directly to the built-in "format()" function. Each formattable +type may define how the format specification is to be interpreted. Most built-in types implement the following options for format specifications, although some of the formatting options are only @@ -10264,10 +10265,10 @@ class is used in a class pattern with positional arguments, each ("u'value'") was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See **PEP 414** for more information. -A string literal with "'f'" or "'F'" in its prefix is a *formatted -string literal*; see f-strings. The "'f'" may be combined with "'r'", -but not with "'b'" or "'u'", therefore raw formatted strings are -possible, but formatted bytes literals are not. +A string literal with "f" or "F" in its prefix is a *formatted string +literal*; see f-strings. The "f" may be combined with "r", but not +with "b" or "u", therefore raw formatted strings are possible, but +formatted bytes literals are not. In triple-quoted literals, unescaped newlines and quotes are allowed (and are retained), except that three unescaped quotes in a row @@ -12703,7 +12704,9 @@ class dict(iterable, **kwargs) | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | @@ -13033,7 +13036,9 @@ class range(start, stop[, step]) | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | diff --git a/Misc/NEWS.d/3.14.0rc1.rst b/Misc/NEWS.d/3.14.0rc1.rst new file mode 100644 index 00000000000000..b7e9a2a7b122fc --- /dev/null +++ b/Misc/NEWS.d/3.14.0rc1.rst @@ -0,0 +1,316 @@ +.. date: 2025-07-05-15-10-42 +.. gh-issue: 136251 +.. nonce: GRM6o8 +.. release date: 2025-07-22 +.. section: Tools/Demos + +Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` + +.. + +.. date: 2025-07-21-14-15-25 +.. gh-issue: 135661 +.. nonce: nAxXw5 +.. section: Security + +Fix parsing attributes with whitespaces around the ``=`` separator in +:class:`html.parser.HTMLParser` according to the HTML5 standard. + +.. + +.. date: 2025-06-09-20-38-25 +.. gh-issue: 118350 +.. nonce: KgWCcP +.. section: Security + +Fix support of escapable raw text mode (elements "textarea" and "title") in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-07-21-22-35-50 +.. gh-issue: 136170 +.. nonce: QUlc78 +.. section: Library + +Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in +3.14.0a7 as it wasn't fully clear which behavior it should have in some +situations so the result was not always what a user might expect. + +.. + +.. date: 2025-07-21-16-10-24 +.. gh-issue: 124621 +.. nonce: wyoWc1 +.. section: Library + +pyrepl now works in Emscripten. + +.. + +.. date: 2025-07-20-16-02-00 +.. gh-issue: 136874 +.. nonce: cLC3o1 +.. section: Library + +Discard URL query and fragment in :func:`urllib.request.url2pathname`. + +.. + +.. date: 2025-07-19-16-20-54 +.. gh-issue: 130645 +.. nonce: O-dYcN +.. section: Library + +Enable color help by default in :mod:`argparse`. + +.. + +.. date: 2025-07-11-23-04-39 +.. gh-issue: 136549 +.. nonce: oAi8u4 +.. section: Library + +Fix signature of :func:`threading.excepthook`. + +.. + +.. date: 2025-07-11-03-39-15 +.. gh-issue: 136523 +.. nonce: s7caKL +.. section: Library + +Fix :class:`wave.Wave_write` emitting an unraisable when open raises. + +.. + +.. date: 2025-07-10-10-18-19 +.. gh-issue: 52876 +.. nonce: 9Vjrd8 +.. section: Library + +Add missing ``keepends`` (default ``True``) parameter to +:meth:`!codecs.StreamReaderWriter.readline` and +:meth:`!codecs.StreamReaderWriter.readlines`. + +.. + +.. date: 2025-07-10-00-47-37 +.. gh-issue: 136470 +.. nonce: KlUEUG +.. section: Library + +Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread +name. + +.. + +.. date: 2025-07-09-20-29-30 +.. gh-issue: 136476 +.. nonce: HyLLzh +.. section: Library + +Fix a bug that was causing the ``get_async_stack_trace`` function to miss +some frames in the stack trace. + +.. + +.. date: 2025-07-08-20-58-01 +.. gh-issue: 136434 +.. nonce: uuJsjS +.. section: Library + +Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` +when running with :option:`-OO`. + +.. + +.. date: 2025-07-07-22-12-32 +.. gh-issue: 136380 +.. nonce: 1b_nXl +.. section: Library + +Raises :exc:`AttributeError` when accessing +:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are +not available. + +.. + +.. date: 2025-06-28-11-32-57 +.. gh-issue: 134759 +.. nonce: AjjKcG +.. section: Library + +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` +when the payload to decode is a :class:`bytes` object. Patch by Kliment +Lamonov. + +.. + +.. date: 2025-05-25-11-02-05 +.. gh-issue: 134657 +.. nonce: 3YFhR9 +.. section: Library + +:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. + +.. + +.. date: 2025-07-19-12-37-05 +.. gh-issue: 136801 +.. nonce: XU_tF2 +.. section: Core and Builtins + +Fix PyREPL syntax highlighting on match cases after multi-line case. +Contributed by Olga Matoula. + +.. + +.. date: 2025-07-12-09-59-14 +.. gh-issue: 136421 +.. nonce: ZD1rNj +.. section: Core and Builtins + +Fix crash when initializing :mod:`datetime` concurrently. + +.. + +.. date: 2025-07-11-13-45-48 +.. gh-issue: 136541 +.. nonce: uZ_-Ju +.. section: Core and Builtins + +Fix some issues with the perf trampolines on x86-64 and aarch64. The +trampolines were not being generated correctly for some cases, which could +lead to the perf integration not working correctly. Patch by Pablo Galindo. + +.. + +.. date: 2025-07-10-23-23-50 +.. gh-issue: 136517 +.. nonce: _NHJyv +.. section: Core and Builtins + +Fixed a typo that prevented printing of uncollectable objects when the +:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. + +.. + +.. date: 2025-07-10-15-53-16 +.. gh-issue: 136525 +.. nonce: xAko0e +.. section: Core and Builtins + +Fix issue where per-thread bytecode was not instrumented for newly created +threads. + +.. + +.. date: 2025-07-08-23-53-51 +.. gh-issue: 132661 +.. nonce: B84iYt +.. section: Core and Builtins + +``Interpolation.expression`` now has a default, the empty string. + +.. + +.. date: 2025-07-08-23-22-08 +.. gh-issue: 132661 +.. nonce: 34ftJl +.. section: Core and Builtins + +Reflect recent :pep:`750` change. + +Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. +Also, disallow implicit concatenation of t-string literals with string or +f-string literals. + +.. + +.. date: 2025-06-12-00-03-34 +.. gh-issue: 116738 +.. nonce: iBBAdo +.. section: Core and Builtins + +Make functions in :mod:`grp` thread-safe on the :term:`free threaded ` build. + +.. + +.. date: 2025-06-06-02-24-42 +.. gh-issue: 135148 +.. nonce: r-t2sC +.. section: Core and Builtins + +Fixed a bug where f-string debug expressions (using =) would incorrectly +strip out parts of strings containing escaped quotes and # characters. Patch +by Pablo Galindo. + +.. + +.. date: 2025-06-03-21-06-22 +.. gh-issue: 133136 +.. nonce: Usnvri +.. section: Core and Builtins + +Limit excess memory usage in the :term:`free threading` build when a large +dictionary or list is resized and accessed by multiple threads. + +.. + +.. date: 2025-05-17-20-56-05 +.. gh-issue: 91153 +.. nonce: afgtG2 +.. section: Core and Builtins + +Fix a crash when a :class:`bytearray` is concurrently mutated during item +assignment. + +.. + +.. date: 2025-04-16-12-01-13 +.. gh-issue: 127971 +.. nonce: pMDOQ0 +.. section: Core and Builtins + +Fix off-by-one read beyond the end of a string in string search. + +.. + +.. date: 2025-07-22-15-18-08 +.. gh-issue: 112068 +.. nonce: 4WvT-8 +.. section: C API + +Revert support of nullable arguments in :c:func:`PyArg_Parse`. + +.. + +.. date: 2025-06-24-11-10-01 +.. gh-issue: 133296 +.. nonce: lIEuVJ +.. section: C API + +New variants for the critical section API that accept one or two +:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now +public in the non-limited C API. + +.. + +.. date: 2025-05-20-17-13-51 +.. gh-issue: 134009 +.. nonce: CpCmry +.. section: C API + +Expose :c:func:`PyMutex_IsLocked` as part of the public C API. + +.. + +.. date: 2025-07-18-17-15-00 +.. gh-issue: 135621 +.. nonce: 9cyCNb +.. section: Build + +PyREPL no longer depends on the :mod:`curses` standard library. Contributed +by Łukasz Langa. diff --git a/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst b/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst deleted file mode 100644 index fe7f962ccbb096..00000000000000 --- a/Misc/NEWS.d/next/Build/2025-07-18-17-15-00.gh-issue-135621.9cyCNb.rst +++ /dev/null @@ -1,2 +0,0 @@ -PyREPL no longer depends on the :mod:`curses` standard library. Contributed -by Łukasz Langa. diff --git a/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst b/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst deleted file mode 100644 index f060f09de19bb8..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-05-20-17-13-51.gh-issue-134009.CpCmry.rst +++ /dev/null @@ -1 +0,0 @@ -Expose :c:func:`PyMutex_IsLocked` as part of the public C API. diff --git a/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst b/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst deleted file mode 100644 index 41401911a6bc0b..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-06-24-11-10-01.gh-issue-133296.lIEuVJ.rst +++ /dev/null @@ -1,3 +0,0 @@ -New variants for the critical section API that accept one or two -:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now -public in the non-limited C API. diff --git a/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst b/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst deleted file mode 100644 index 018c5c7880c001..00000000000000 --- a/Misc/NEWS.d/next/C_API/2025-07-22-15-18-08.gh-issue-112068.4WvT-8.rst +++ /dev/null @@ -1 +0,0 @@ -Revert support of nullable arguments in :c:func:`PyArg_Parse`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst deleted file mode 100644 index ced7a9c9fd3e63..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-04-16-12-01-13.gh-issue-127971.pMDOQ0.rst +++ /dev/null @@ -1 +0,0 @@ -Fix off-by-one read beyond the end of a string in string search. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst deleted file mode 100644 index dc2f1e22ba5b05..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-05-17-20-56-05.gh-issue-91153.afgtG2.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a crash when a :class:`bytearray` is concurrently mutated during item assignment. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst deleted file mode 100644 index a9501c13c95b3a..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-03-21-06-22.gh-issue-133136.Usnvri.rst +++ /dev/null @@ -1,2 +0,0 @@ -Limit excess memory usage in the :term:`free threading` build when a -large dictionary or list is resized and accessed by multiple threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst deleted file mode 100644 index 9b1f62433b45ed..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-06-02-24-42.gh-issue-135148.r-t2sC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug where f-string debug expressions (using =) would incorrectly -strip out parts of strings containing escaped quotes and # characters. Patch -by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst deleted file mode 100644 index 2a1ed2944d8ddf..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-06-12-00-03-34.gh-issue-116738.iBBAdo.rst +++ /dev/null @@ -1 +0,0 @@ -Make functions in :mod:`grp` thread-safe on the :term:`free threaded ` build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst deleted file mode 100644 index 5d59e024389c18..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-22-08.gh-issue-132661.34ftJl.rst +++ /dev/null @@ -1,5 +0,0 @@ -Reflect recent :pep:`750` change. - -Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. -Also, disallow implicit concatenation of t-string literals with string or -f-string literals. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst deleted file mode 100644 index 9930413b53c150..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-08-23-53-51.gh-issue-132661.B84iYt.rst +++ /dev/null @@ -1 +0,0 @@ -``Interpolation.expression`` now has a default, the empty string. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst deleted file mode 100644 index f28eb2ca3b71e8..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-15-53-16.gh-issue-136525.xAko0e.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix issue where per-thread bytecode was not instrumented for newly created -threads. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst deleted file mode 100644 index bf26c4eb0e4c74..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-10-23-23-50.gh-issue-136517._NHJyv.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a typo that prevented printing of uncollectable objects when the -:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst deleted file mode 100644 index af9b94ad0613d7..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-11-13-45-48.gh-issue-136541.uZ_-Ju.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix some issues with the perf trampolines on x86-64 and aarch64. The -trampolines were not being generated correctly for some cases, which could -lead to the perf integration not working correctly. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst deleted file mode 100644 index dcc73267a78546..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-12-09-59-14.gh-issue-136421.ZD1rNj.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash when initializing :mod:`datetime` concurrently. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst b/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst deleted file mode 100644 index 767d7b97726971..00000000000000 --- a/Misc/NEWS.d/next/Core_and_Builtins/2025-07-19-12-37-05.gh-issue-136801.XU_tF2.rst +++ /dev/null @@ -1 +0,0 @@ -Fix PyREPL syntax highlighting on match cases after multi-line case. Contributed by Olga Matoula. diff --git a/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst b/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst deleted file mode 100644 index 1bf8ee504ef180..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-05-25-11-02-05.gh-issue-134657.3YFhR9.rst +++ /dev/null @@ -1 +0,0 @@ -:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. diff --git a/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst b/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst deleted file mode 100644 index 79b85320926954..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-06-28-11-32-57.gh-issue-134759.AjjKcG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` when -the payload to decode is a :class:`bytes` object. Patch by Kliment Lamonov. diff --git a/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst b/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst deleted file mode 100644 index 4ac04b9c0a6c5c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-07-22-12-32.gh-issue-136380.1b_nXl.rst +++ /dev/null @@ -1,3 +0,0 @@ -Raises :exc:`AttributeError` when accessing -:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are -not available. diff --git a/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst b/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst deleted file mode 100644 index 951f57100b650c..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-08-20-58-01.gh-issue-136434.uuJsjS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` -when running with :option:`-OO`. diff --git a/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst b/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst deleted file mode 100644 index 7634bd3be9346d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-09-20-29-30.gh-issue-136476.HyLLzh.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix a bug that was causing the ``get_async_stack_trace`` function to miss -some frames in the stack trace. diff --git a/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst b/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst deleted file mode 100644 index 5a0429cae07168..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-10-00-47-37.gh-issue-136470.KlUEUG.rst +++ /dev/null @@ -1,2 +0,0 @@ -Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread -name. diff --git a/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst b/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst deleted file mode 100644 index a835306868d3dc..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-10-10-18-19.gh-issue-52876.9Vjrd8.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add missing ``keepends`` (default ``True``) parameter to -:meth:`!codecs.StreamReaderWriter.readline` and -:meth:`!codecs.StreamReaderWriter.readlines`. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst b/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst deleted file mode 100644 index 71ec66a37ef4c3..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-11-03-39-15.gh-issue-136523.s7caKL.rst +++ /dev/null @@ -1 +0,0 @@ -Fix :class:`wave.Wave_write` emitting an unraisable when open raises. diff --git a/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst b/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst deleted file mode 100644 index f3050ad5d5aa10..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-11-23-04-39.gh-issue-136549.oAi8u4.rst +++ /dev/null @@ -1 +0,0 @@ -Fix signature of :func:`threading.excepthook`. diff --git a/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst b/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst deleted file mode 100644 index 96e076dfe5bd12..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-19-16-20-54.gh-issue-130645.O-dYcN.rst +++ /dev/null @@ -1 +0,0 @@ -Enable color help by default in :mod:`argparse`. diff --git a/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst b/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst deleted file mode 100644 index 9a71eb8ef1ac8d..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-20-16-02-00.gh-issue-136874.cLC3o1.rst +++ /dev/null @@ -1 +0,0 @@ -Discard URL query and fragment in :func:`urllib.request.url2pathname`. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst b/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst deleted file mode 100644 index 34049183649271..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-21-16-10-24.gh-issue-124621.wyoWc1.rst +++ /dev/null @@ -1 +0,0 @@ -pyrepl now works in Emscripten. diff --git a/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst b/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst deleted file mode 100644 index fd30fe156a1b32..00000000000000 --- a/Misc/NEWS.d/next/Library/2025-07-21-22-35-50.gh-issue-136170.QUlc78.rst +++ /dev/null @@ -1,3 +0,0 @@ -Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in 3.14.0a7 -as it wasn't fully clear which behavior it should have in some situations so -the result was not always what a user might expect. diff --git a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst b/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst deleted file mode 100644 index 6ad3caf33b2201..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-06-09-20-38-25.gh-issue-118350.KgWCcP.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix support of escapable raw text mode (elements "textarea" and "title") -in :class:`html.parser.HTMLParser`. diff --git a/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst b/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst deleted file mode 100644 index 533e4df8626b90..00000000000000 --- a/Misc/NEWS.d/next/Security/2025-07-21-14-15-25.gh-issue-135661.nAxXw5.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix parsing attributes with whitespaces around the ``=`` separator in -:class:`html.parser.HTMLParser` according to the HTML5 standard. diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst b/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst deleted file mode 100644 index 6a35afe15e3875..00000000000000 --- a/Misc/NEWS.d/next/Tools-Demos/2025-07-05-15-10-42.gh-issue-136251.GRM6o8.rst +++ /dev/null @@ -1 +0,0 @@ -Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` diff --git a/README.rst b/README.rst index 0092dfb497a021..63211550bc6654 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.14.0 beta 4 -==================================== +This is Python version 3.14.0 release candidate 1 +================================================= .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions From bb657b2dc9760dd2b64812aff5048a84da6853ce Mon Sep 17 00:00:00 2001 From: Haolin Qin Date: Thu, 31 Jul 2025 20:54:49 +0800 Subject: [PATCH 713/713] Fix outdated wording (#137232) --- Doc/howto/free-threading-python.rst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index 24069617c47ae1..cd904d0e567b7f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -120,9 +120,8 @@ objects of these types may see increased memory usage. This is expected to be addressed in the 3.14 release. Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. - +returned by :func:`sys.intern` are also immortalized. This behavior is retained +in the 3.14 free-threaded build. Frame objects ------------- @@ -150,8 +149,8 @@ compared to the default GIL-enabled build. In 3.13, this overhead is about 40% on the `pyperformance `_ suite. Programs that spend most of their time in C extensions or I/O will see less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is +interpreter (:pep:`659`) is disabled in the free-threaded build. We +re-enabled it in a thread-safe way in the 3.14 release. This overhead is expected to be reduced in upcoming Python release. We are aiming for an overhead of 10% or less on the pyperformance suite compared to the default GIL-enabled build.