diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f8c468475470c..8787402ccc4423 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,6 +58,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 1 + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Check Autoconf and aclocal versions @@ -94,6 +95,8 @@ jobs: if: needs.check_source.outputs.run_tests == 'true' steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.x' @@ -120,7 +123,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom + make regen-stdlib-module-names regen-sbom regen-unicodedata - name: Check for changes run: | git add -u @@ -235,10 +238,19 @@ jobs: free-threading: - false - true + os: + - ubuntu-24.04 + - ubuntu-24.04-aarch64 + is-fork: # only used for the exclusion trick + - ${{ github.repository_owner != 'python' }} + exclude: + - os: ubuntu-24.04-aarch64 + is-fork: true uses: ./.github/workflows/reusable-ubuntu.yml with: config_hash: ${{ needs.check_source.outputs.config_hash }} free-threading: ${{ matrix.free-threading }} + os: ${{ matrix.os }} build_ubuntu_ssltests: name: 'Ubuntu SSL tests with OpenSSL' @@ -259,6 +271,8 @@ jobs: LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache @@ -319,6 +333,8 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install Dependencies @@ -402,7 +418,7 @@ jobs: # # (GH-104097) test_sysconfig is skipped because it has tests that are # failing when executed from inside a virtual environment. - ${{ env.VENV_PYTHON }} -m test \ + "${VENV_PYTHON}" -m test \ -W \ -o \ -j4 \ @@ -437,6 +453,8 @@ jobs: ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml index 43a7afec73884e..fdb4b9aa29a7c8 100644 --- a/.github/workflows/documentation-links.yml +++ b/.github/workflows/documentation-links.yml @@ -10,9 +10,6 @@ on: - 'Doc/**' - '.github/workflows/doc.yml' -permissions: - pull-requests: write - concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true @@ -20,6 +17,9 @@ concurrency: jobs: documentation-links: runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: - uses: readthedocs/actions/preview@v1 with: diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 7dbbe71b2131e7..9b84998a55666d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -28,10 +28,12 @@ concurrency: jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 timeout-minutes: 90 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build tier two interpreter run: | ./configure --enable-experimental-jit=interpreter --with-pydebug @@ -78,12 +80,15 @@ jobs: runner: macos-14 - target: x86_64-unknown-linux-gnu/gcc architecture: x86_64 - runner: ubuntu-22.04 + runner: ubuntu-24.04 - target: aarch64-unknown-linux-gnu/gcc architecture: aarch64 - runner: ubuntu-22.04 + # Forks don't have access to our paid AArch64 runners. These jobs are skipped below: + runner: ${{ github.repository_owner == 'python' && 'ubuntu-24.04-aarch64' || 'ubuntu-24.04' }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.11' @@ -104,7 +109,7 @@ jobs: # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes + # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - name: Native macOS if: runner.os == 'macOS' @@ -118,7 +123,8 @@ jobs: ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - name: Native Linux - if: runner.os == 'Linux' && matrix.architecture == 'x86_64' + # Forks don't have access to our paid AArch64 runners. Skip those: + if: runner.os == 'Linux' && (matrix.architecture == 'x86_64' || github.repository_owner == 'python') run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" @@ -126,35 +132,18 @@ jobs: make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Emulated Linux - if: runner.os == 'Linux' && matrix.architecture != 'x86_64' - # The --ignorefile on ./python -m test is used to exclude tests known to fail when running on an emulated Linux. - run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - ./configure --prefix="$(pwd)/../build" - make install --jobs 4 - make clean --jobs 4 - export HOST=${{ matrix.architecture }}-linux-gnu - sudo apt install --yes "gcc-$HOST" qemu-user - export QEMU_LD_PREFIX="/usr/$HOST" - CC="$HOST-gcc" \ - CPP="$HOST-gcc --preprocess" \ - HOSTRUNNER=qemu-${{ matrix.architecture }} \ - ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} --build=x86_64-linux-gnu --host="$HOST" --with-build-python=../build/bin/python3 --with-pkg-config=no ac_cv_buggy_getaddrinfo=no ac_cv_file__dev_ptc=no ac_cv_file__dev_ptmx=yes - make all --jobs 4 - ./python -m test --ignorefile=Tools/jit/ignore-tests-emulated-linux.txt --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit-with-disabled-gil: name: Free-Threaded (Debug) needs: interpreter - runs-on: ubuntu-22.04 + runs-on: ubuntu-24.04 strategy: matrix: llvm: - 19 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3.11' diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index ccde03f91983df..d74ce8fcc256dc 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -20,6 +20,8 @@ jobs: steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.x" diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index e5b05302b5ac27..5dfa8d7bcafd78 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -51,6 +51,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: "3.13" diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index bbedd22cc6d189..0a6277c779ff67 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,15 +4,14 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] -permissions: - issues: write - pull-requests: write - jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write timeout-minutes: 10 steps: @@ -28,6 +27,9 @@ jobs: name: Unresolved review if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml index 1a6fd33186840c..964bd87e815f42 100644 --- a/.github/workflows/reusable-change-detection.yml +++ b/.github/workflows/reusable-change-detection.yml @@ -61,6 +61,8 @@ jobs: - run: >- echo '${{ github.event_name }}' - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Check for source changes id: check run: | diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 39a97392e898aa..3962d12403919a 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -22,12 +22,14 @@ jobs: env: branch_base: 'origin/${{ github.event.pull_request.base.ref }}' branch_pr: 'origin/${{ github.event.pull_request.head.ref }}' + commits: ${{ github.event.pull_request.commits }} refspec_base: '+${{ github.event.pull_request.base.sha }}:remotes/origin/${{ github.event.pull_request.base.ref }}' refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' uses: actions/checkout@v4 with: + persist-credentials: false ref: >- ${{ github.event_name == 'pull_request' @@ -39,15 +41,15 @@ jobs: if: github.event_name == 'pull_request' run: | # Fetch enough history to find a common ancestor commit (aka merge-base): - git fetch origin ${{ env.refspec_pr }} --depth=$(( ${{ github.event.pull_request.commits }} + 1 )) \ + git fetch origin "${refspec_pr}" --depth=$(( commits + 1 )) \ --no-tags --prune --no-recurse-submodules # This should get the oldest commit in the local fetched history (which may not be the commit the PR branched from): - COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 ${{ env.branch_pr }} ) + COMMON_ANCESTOR=$( git rev-list --first-parent --max-parents=0 --max-count=1 "${branch_pr}" ) DATE=$( git log --date=iso8601 --format=%cd "${COMMON_ANCESTOR}" ) # Get all commits since that commit date from the base branch (eg: master or main): - git fetch origin ${{ env.refspec_base }} --shallow-since="${DATE}" \ + git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' uses: actions/setup-python@v5 @@ -69,7 +71,7 @@ jobs: if: github.event_name == 'pull_request' run: | python Doc/tools/check-warnings.py \ - --annotate-diff '${{ env.branch_base }}' '${{ env.branch_pr }}' \ + --annotate-diff "${branch_base}" "${branch_pr}" \ --fail-if-regression \ --fail-if-improved \ --fail-if-new-news-nit @@ -81,6 +83,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: 'Set up Python' uses: actions/setup-python@v5 with: @@ -99,6 +103,8 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/cache@v4 with: path: ~/.cache/pip diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index 4c3dd10194f8cb..36ae3e27207e37 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -29,6 +29,8 @@ jobs: runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml index 7a4d81f0bdcad1..b5144ca3e9efc4 100644 --- a/.github/workflows/reusable-tsan.yml +++ b/.github/workflows/reusable-tsan.yml @@ -23,8 +23,13 @@ jobs: name: 'Thread sanitizer' runs-on: ubuntu-24.04 timeout-minutes: 60 + env: + OPTIONS: ${{ inputs.options }} + SUPPRESSIONS_PATH: ${{ inputs.suppressions_path }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Runner image version run: echo "IMAGE_VERSION=${ImageVersion}" >> "$GITHUB_ENV" - name: Restore config.cache @@ -47,7 +52,7 @@ jobs: sudo sysctl -w vm.mmap_rnd_bits=28 - name: TSAN Option Setup run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${{ inputs.suppressions_path }} handle_segv=0" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/${SUPPRESSIONS_PATH} handle_segv=0" >> "$GITHUB_ENV" echo "CC=clang" >> "$GITHUB_ENV" echo "CXX=clang++" >> "$GITHUB_ENV" - name: Add ccache to PATH @@ -59,7 +64,7 @@ jobs: save: ${{ github.event_name == 'push' }} max-size: "200M" - name: Configure CPython - run: ${{ inputs.options }} + run: "${OPTIONS}" - name: Build CPython run: make -j4 - name: Display build info diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 642354f8b4f61b..46c542940c8483 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -11,16 +11,16 @@ on: required: false type: boolean default: false + os: + description: OS to run the job + required: true + type: string jobs: build_ubuntu_reusable: - name: 'build and test' + name: build and test (${{ inputs.os }}) timeout-minutes: 60 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-24.04, ubuntu-24.04-aarch64] + runs-on: ${{ inputs.os }} env: FORCE_COLOR: 1 OPENSSL_VER: 3.0.15 @@ -28,6 +28,8 @@ jobs: TERM: linux steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies @@ -42,7 +44,7 @@ jobs: uses: actions/cache@v4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux @@ -84,7 +86,7 @@ jobs: working-directory: ${{ env.CPYTHON_BUILDDIR }} run: make -j - name: Build CPython out-of-tree (for compiler warning check) - if: ${{ !inputs.free-threading}} + if: ${{ !inputs.free-threading }} working-directory: ${{ env.CPYTHON_BUILDDIR }} run: set -o pipefail; make -j --output-sync 2>&1 | tee compiler_output_ubuntu.txt - name: Display build info @@ -94,7 +96,7 @@ jobs: if: ${{ !inputs.free-threading }} run: >- python Tools/build/check_warnings.py - --compiler-output-file-path=${{ env.CPYTHON_BUILDDIR }}/compiler_output_ubuntu.txt + --compiler-output-file-path="${CPYTHON_BUILDDIR}/compiler_output_ubuntu.txt" --warning-ignore-file-path "${GITHUB_WORKSPACE}/Tools/build/.warningignore_ubuntu" --compiler-output-type=gcc --fail-on-regression diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 3f96c888e2dd30..4356d9c1c8795e 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -20,6 +20,8 @@ jobs: CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" uses: bytecodealliance/actions/wasmtime/setup@v1 @@ -34,9 +36,9 @@ jobs: - name: "Install WASI SDK" # Hard-coded to x64. if: steps.cache-wasi-sdk.outputs.cache-hit != 'true' run: | - mkdir ${{ env.WASI_SDK_PATH }} && \ - curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${{ env.WASI_SDK_VERSION }}/wasi-sdk-${{ env.WASI_SDK_VERSION }}.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip + mkdir "${WASI_SDK_PATH}" && \ + curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ + tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - name: "Configure ccache action" uses: hendrikmuhs/ccache-action@v1.2 with: @@ -72,6 +74,6 @@ jobs: - name: "Make host" run: python3 Tools/wasm/wasi.py make-host - name: "Display build info" - run: make --directory ${{ env.CROSS_BUILD_WASI }} pythoninfo + run: make --directory "${CROSS_BUILD_WASI}" pythoninfo - name: "Test" - run: make --directory ${{ env.CROSS_BUILD_WASI }} test + run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index abdb1a1982fef8..a1c45d954247fb 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,8 +17,12 @@ jobs: runs-on: windows-latest timeout-minutes: 60 env: + ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Build CPython installer - run: .\Tools\msi\build.bat --doc -${{ inputs.arch }} + run: ./Tools/msi/build.bat --doc -"${ARCH}" + shell: bash diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 12b68d68466d62..459d2b29e5d42b 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -26,8 +26,12 @@ jobs: name: 'build and test (${{ inputs.arch }})' runs-on: ${{ inputs.os }} timeout-minutes: 60 + env: + ARCH: ${{ inputs.arch }} steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - name: Register MSVC problem matcher if: inputs.arch != 'Win32' run: echo "::add-matcher::.github/problem-matchers/msvc.json" @@ -35,8 +39,9 @@ jobs: run: >- .\\PCbuild\\build.bat -e -d -v - -p ${{ inputs.arch }} + -p "${ARCH}" ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash - name: Display build info # FIXME(diegorusso): remove the `if` if: inputs.arch != 'arm64' run: .\\python.bat -m test.pythoninfo @@ -44,6 +49,7 @@ jobs: if: inputs.arch != 'arm64' run: >- .\\PCbuild\\rt.bat - -p ${{ inputs.arch }} + -p "${ARCH}" -d -q --fast-ci ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + shell: bash diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f97587e68cbbe4..7578189f5d4d67 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,14 +4,13 @@ on: schedule: - cron: "0 */6 * * *" -permissions: - pull-requests: write - jobs: stale: if: github.repository_owner == 'python' runs-on: ubuntu-latest + permissions: + pull-requests: write timeout-minutes: 10 steps: diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 83b007f1c9c2ef..463e7bf3355cc3 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -26,6 +26,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/setup-python@v5 with: python-version: '3' diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 00000000000000..eeda8d9eaaf484 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,6 @@ +# Configuration for the zizmor static analysis tool, run via pre-commit in CI +# https://woodruffw.github.io/zizmor/configuration/ +rules: + dangerous-triggers: + ignore: + - documentation-links.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ccaf2390d99fae..107f3b255735f4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.7.1 + rev: v0.8.2 hooks: - id: ruff name: Run Ruff (lint) on Doc/ @@ -51,7 +51,7 @@ repos: types_or: [c, inc, python, rst] - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.4 + rev: 0.30.0 hooks: - id: check-dependabot - id: check-github-workflows @@ -61,6 +61,11 @@ repos: hooks: - id: actionlint + - repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v0.8.0 + hooks: + - id: zizmor + - repo: https://github.com/sphinx-contrib/sphinx-lint rev: v1.0.0 hooks: diff --git a/Doc/Makefile b/Doc/Makefile index 22e43ee3e542ee..4a704ad58b33d3 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -144,7 +144,7 @@ pydoc-topics: build .PHONY: gettext gettext: BUILDER = gettext -gettext: SPHINXOPTS += -d build/doctrees-gettext +gettext: override SPHINXOPTS := -d build/doctrees-gettext $(SPHINXOPTS) gettext: build .PHONY: htmlview diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index fc2336d120c259..c1f0bd750361d6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -853,12 +853,23 @@ The following functions are used to create and modify Unicode exceptions from C. *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *start* is ``0``. Otherwise, it is clipped to ``[0, len(object) - 1]``. + + .. seealso:: :attr:`UnicodeError.start` + .. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. + Set the *start* attribute of the given exception object to *start*. + Return ``0`` on success, ``-1`` on failure. + + .. note:: + + While passing a negative *start* does not raise an exception, + the corresponding getters will not consider it as a relative + offset. .. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) @@ -868,6 +879,9 @@ The following functions are used to create and modify Unicode exceptions from C. *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on failure. + If the :attr:`UnicodeError.object` is an empty sequence, the resulting + *end* is ``0``. Otherwise, it is clipped to ``[1, len(object)]``. + .. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) @@ -875,6 +889,8 @@ The following functions are used to create and modify Unicode exceptions from C. Set the *end* attribute of the given exception object to *end*. Return ``0`` on success, ``-1`` on failure. + .. seealso:: :attr:`UnicodeError.end` + .. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 638a740e0c24da..1a52e146a69751 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -132,7 +132,7 @@ See also :ref:`Reflection `. .. versionadded:: 3.11 .. versionchanged:: 3.13 - As part of :pep:`667`, return a proxy object for optimized scopes. + As part of :pep:`667`, return an instance of :c:var:`PyFrameLocalsProxy_Type`. .. c:function:: int PyFrame_GetLineNumber(PyFrameObject *frame) @@ -140,6 +140,26 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. +Frame Locals Proxies +^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 3.13 + +The :attr:`~frame.f_locals` attribute on a :ref:`frame object ` +is an instance of a "frame-locals proxy". The proxy object exposes a +write-through view of the underlying locals dictionary for the frame. This +ensures that the variables exposed by ``f_locals`` are always up to date with +the live local variables in the frame itself. + +See :pep:`667` for more information. + +.. c:var:: PyTypeObject PyFrameLocalsProxy_Type + + The type of frame :func:`locals` proxy objects. + +.. c:function:: int PyFrameLocalsProxy_Check(PyObject *obj) + + Return non-zero if *obj* is a frame :func:`locals` proxy. Internal Frames ^^^^^^^^^^^^^^^ diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index ba1c2852f0bd53..dd63dd013e32dc 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -567,6 +567,15 @@ Initializing and finalizing the interpreter customized Python that always runs in isolated mode using :c:func:`Py_RunMain`. +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + The :term:`GIL` must be held for *interp*. + + .. versionadded:: 3.13 Process-wide parameters ======================= diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index d6569ddcf586fa..6b33d93a9f2af9 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -1281,6 +1281,17 @@ PyConfig Default: ``1`` in Python config and ``0`` in isolated config. + .. c:member:: int use_system_logger + + If non-zero, ``stdout`` and ``stderr`` will be redirected to the system + log. + + Only available on macOS 10.12 and later, and on iOS. + + Default: ``0`` (don't use system log). + + .. versionadded:: 3.13.2 + .. c:member:: int user_site_directory If non-zero, add the user site directory to :data:`sys.path`. diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 2ac975ff7d1a59..1ae3c46bea46ea 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -85,7 +85,7 @@ Object Protocol instead of the :func:`repr`. -.. c:function:: int PyObject_HasAttrWithError(PyObject *o, const char *attr_name) +.. c:function:: int PyObject_HasAttrWithError(PyObject *o, PyObject *attr_name) Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This is equivalent to the Python expression ``hasattr(o, attr_name)``. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 5b9e43874c7f2b..124e58cf950b7a 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -66,7 +66,7 @@ Limited C API Python 3.2 introduced the *Limited API*, a subset of Python's C API. Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. +compiled once and be loaded on multiple versions of Python. Contents of the Limited API are :ref:`listed below `. .. c:macro:: Py_LIMITED_API @@ -76,7 +76,7 @@ Contents of the Limited API are :ref:`listed below `. Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases + The extension will be ABI-compatible with all Python 3 releases from the specified one onward, and can use Limited API introduced up to that version. @@ -94,7 +94,15 @@ Stable ABI ---------- To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. +remain ABI-compatible across Python 3.x versions. + +.. note:: + + The Stable ABI prevents ABI issues, like linker errors due to missing + symbols or data corruption due to changes in structure layouts or function + signatures. + However, other changes in Python can change the *behavior* of extensions. + See Python's Backwards Compatibility Policy (:pep:`387`) for details. The Stable ABI contains symbols exposed in the :ref:`Limited API `, but also other ones – for example, functions necessary to diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index d6fca1a0b0a219..c688afdca8231d 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -426,3 +426,7 @@ Process Control function registered last is called first. Each cleanup function will be called at most once. Since Python's internal finalization will have completed before the cleanup function, no Python APIs should be called by *func*. + + .. seealso:: + + :c:func:`PyUnstable_AtExit` for passing a ``void *data`` argument. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 86d3967d9fb577..444b3456f051d8 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -529,19 +529,19 @@ The following functions and structs are used to create The following “offset” fields cannot be set using :c:type:`PyType_Slot`: - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) - * :c:member:`~PyTypeObject.tp_dictoffset` - (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (use ``"__vectorcalloffset__"`` in - :ref:`PyMemberDef `) - - If it is not possible to switch to a ``MANAGED`` flag (for example, - for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. - See :ref:`PyMemberDef documentation ` - for details. + * :c:member:`~PyTypeObject.tp_weaklistoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_WEAKREF` instead if possible) + * :c:member:`~PyTypeObject.tp_dictoffset` + (use :c:macro:`Py_TPFLAGS_MANAGED_DICT` instead if possible) + * :c:member:`~PyTypeObject.tp_vectorcall_offset` + (use ``"__vectorcalloffset__"`` in + :ref:`PyMemberDef `) + + If it is not possible to switch to a ``MANAGED`` flag (for example, + for vectorcall or to support Python older than 3.12), specify the + offset in :c:member:`Py_tp_members `. + See :ref:`PyMemberDef documentation ` + for details. The following internal fields cannot be set at all when creating a heap type: @@ -557,20 +557,18 @@ The following functions and structs are used to create To avoid issues, use the *bases* argument of :c:func:`PyType_FromSpecWithBases` instead. - .. versionchanged:: 3.9 - - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. + .. versionchanged:: 3.9 + Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. + .. versionchanged:: 3.11 + :c:member:`~PyBufferProcs.bf_getbuffer` and + :c:member:`~PyBufferProcs.bf_releasebuffer` are now available + under the :ref:`limited API `. - .. versionchanged:: 3.14 - - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation - for details. + .. versionchanged:: 3.14 + The field :c:member:`~PyTypeObject.tp_vectorcall` can now set + using ``Py_tp_vectorcall``. See the field's documentation + for details. .. c:member:: void *pfunc diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 59bd7661965d93..dcbc8804cd6b89 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -1588,6 +1588,11 @@ object. Create a Unicode writer instance. + *length* must be greater than or equal to ``0``. + + If *length* is greater than ``0``, preallocate an internal buffer of + *length* characters. + Set an exception and return ``NULL`` on error. .. c:function:: PyObject* PyUnicodeWriter_Finish(PyUnicodeWriter *writer) @@ -1596,12 +1601,16 @@ object. Set an exception and return ``NULL`` on error. + The writer instance is invalid after this call. + .. c:function:: void PyUnicodeWriter_Discard(PyUnicodeWriter *writer) Discard the internal Unicode buffer and destroy the writer instance. If *writer* is ``NULL``, no operation is performed. + The writer instance is invalid after this call. + .. c:function:: int PyUnicodeWriter_WriteChar(PyUnicodeWriter *writer, Py_UCS4 ch) Write the single Unicode character *ch* into *writer*. diff --git a/Doc/conf.py b/Doc/conf.py index 738c9901eef06f..9cde394cbaed69 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -78,6 +78,13 @@ .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` + +.. Apparently this how you hack together a formatted link: + (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) +.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` +.. _FORCE_COLOR: https://force-color.org/ +.. |NO_COLOR| replace:: ``NO_COLOR`` +.. _NO_COLOR: https://no-color.org/ """ # There are two options for replacing |today|. Either, you set today to some diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 6bfcc191b2270b..3f49c88c3cc028 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -180,7 +180,7 @@ PyCapsule_IsValid:const char*:name:: PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: +PyCapsule_New:void (*)(PyObject *):destructor:: PyCapsule_SetContext:int::: PyCapsule_SetContext:PyObject*:self:0: @@ -349,11 +349,11 @@ PyComplex_CheckExact:int::: PyComplex_CheckExact:PyObject*:p:0: PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: +PyComplex_FromCComplex:Py_complex:v:: PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: +PyComplex_FromDoubles:double:real:: +PyComplex_FromDoubles:double:imag:: PyComplex_ImagAsDouble:double::: PyComplex_ImagAsDouble:PyObject*:op:0: @@ -620,7 +620,9 @@ PyErr_GetExcInfo:PyObject**:pvalue:+1: PyErr_GetExcInfo:PyObject**:ptraceback:+1: PyErr_GetRaisedException:PyObject*::+1: -PyErr_SetRaisedException:::: + +PyErr_SetRaisedException:void::: +PyErr_SetRaisedException:PyObject *:exc:0:stolen PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: @@ -640,9 +642,9 @@ PyErr_NewExceptionWithDoc:PyObject*:dict:0: PyErr_NoMemory:PyObject*::null: PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? +PyErr_NormalizeException:PyObject**:exc:+1:??? +PyErr_NormalizeException:PyObject**:val:+1:??? +PyErr_NormalizeException:PyObject**:tb:+1:??? PyErr_Occurred:PyObject*::0: @@ -1314,7 +1316,7 @@ PyMapping_GetItemString:const char*:key:: PyMapping_HasKey:int::: PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: +PyMapping_HasKey:PyObject*:key:0: PyMapping_HasKeyString:int::: PyMapping_HasKeyString:PyObject*:o:0: @@ -1474,7 +1476,7 @@ PyModule_GetState:void*::: PyModule_GetState:PyObject*:module:0: PyModule_New:PyObject*::+1: -PyModule_New::char* name:: +PyModule_New:char*:name:: PyModule_NewObject:PyObject*::+1: PyModule_NewObject:PyObject*:name:+1: @@ -1484,7 +1486,7 @@ PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: +PyModuleDef_Init:PyModuleDef*:def:: PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -1984,10 +1986,10 @@ PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: PySeqIter_Check:int::: -PySeqIter_Check::op:: +PySeqIter_Check:PyObject *:op:0: PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: +PySeqIter_New:PyObject*:seq:0: PySequence_Check:int::: PySequence_Check:PyObject*:o:0: @@ -2421,7 +2423,7 @@ PyUnicode_GET_LENGTH:PyObject*:o:0: PyUnicode_KIND:int::: PyUnicode_KIND:PyObject*:o:0: -PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:Py_UCS4::: PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: Py_UNICODE_ISALNUM:int::: @@ -2488,7 +2490,7 @@ PyUnicode_FromWideChar:const wchar_t*:w:: PyUnicode_FromWideChar:Py_ssize_t:size:: PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: +PyUnicode_AsWideChar:PyObject*:unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: PyUnicode_AsWideChar:Py_ssize_t:size:: @@ -2541,7 +2543,7 @@ PyUnicode_AsUTF8String:PyObject*:unicode:0: PyUnicode_AsUTF8AndSize:const char*::: PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:: PyUnicode_AsUTF8:const char*::: PyUnicode_AsUTF8:PyObject*:unicode:0: @@ -2864,13 +2866,13 @@ PyUnicodeDecodeError_SetStart:PyObject*:exc:0: PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: +PyWeakref_Check:PyObject*:ob:0: PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: +PyWeakref_CheckProxy:PyObject*:ob:0: PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: +PyWeakref_CheckRef:PyObject*:ob:0: PyWeakref_GET_OBJECT:PyObject*::0: PyWeakref_GET_OBJECT:PyObject*:ref:0: diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 6abe93d71ad529..c1ad42e7e55ee5 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -167,6 +167,8 @@ that return :term:`strong references `. +-----------------------------------+-----------------------------------+ | :c:func:`PyImport_AddModule` | :c:func:`PyImport_AddModuleRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyCell_GET` | :c:func:`PyCell_Get` | ++-----------------------------------+-----------------------------------+ Not all APIs that return borrowed references are problematic. For example, :c:func:`PyTuple_GetItem` is safe because tuples are immutable. diff --git a/Doc/howto/gdb_helpers.rst b/Doc/howto/gdb_helpers.rst index 53bbf7ddaa2ab9..98ce813ca4ab02 100644 --- a/Doc/howto/gdb_helpers.rst +++ b/Doc/howto/gdb_helpers.rst @@ -180,7 +180,7 @@ regular machine-level integer:: (gdb) p some_python_integer $4 = 42 -The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`: +The internal structure can be revealed with a cast to :c:expr:`PyLongObject *`:: (gdb) p *(PyLongObject*)some_python_integer $5 = {ob_base = {ob_base = {ob_refcnt = 8, ob_type = 0x3dad39f5e0}, ob_size = 1}, diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 3182d5664ab6ec..2982cf88bf97b4 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -127,7 +127,7 @@ that; formatting options will also be explained later. Notice that in this example, we use functions directly on the ``logging`` module, like ``logging.debug``, rather than creating a logger and calling -functions on it. These functions operation on the root logger, but can be useful +functions on it. These functions operate on the root logger, but can be useful as they will call :func:`~logging.basicConfig` for you if it has not been called yet, like in this example. In larger programs you'll usually want to control the logging configuration explicitly however - so for that reason as well as others, it's diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 3cf8e2737e85dc..77c2e97da11990 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -259,8 +259,8 @@ Condition Note that a task *may* return from this call spuriously, which is why the caller should always re-check the state - and be prepared to :meth:`wait` again. For this reason, you may - prefer to use :meth:`wait_for` instead. + and be prepared to :meth:`~Condition.wait` again. For this reason, you may + prefer to use :meth:`~Condition.wait_for` instead. .. coroutinemethod:: wait_for(predicate) @@ -268,7 +268,7 @@ Condition The predicate must be a callable which result will be interpreted as a boolean value. The method will repeatedly - :meth:`wait` until the predicate evaluates to *true*. The final value is the + :meth:`~Condition.wait` until the predicate evaluates to *true*. The final value is the return value. @@ -434,7 +434,7 @@ Barrier .. coroutinemethod:: abort() Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + calls to :meth:`~Barrier.wait` to fail with the :class:`BrokenBarrierError`. Use this for example if one of the tasks needs to abort, to avoid infinite waiting tasks. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 0cc9063f153aba..5b4e445762e076 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -783,10 +783,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` +function which returns an empty :class:`list`. The :meth:`!list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is +:meth:`!list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index bd9529db9ee65a..bd88fa377fb39d 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -1949,6 +1949,24 @@ Utility functions It behaves similar to ``pointer(obj)``, but the construction is a lot faster. +.. function:: CopyComPointer(src, dst) + + Copies a COM pointer from *src* to *dst* and returns the Windows specific + :c:type:`!HRESULT` value. + + If *src* is not ``NULL``, its ``AddRef`` method is called, incrementing the + reference count. + + In contrast, the reference count of *dst* will not be decremented before + assigning the new value. Unless *dst* is ``NULL``, the caller is responsible + for decrementing the reference count by calling its ``Release`` method when + necessary. + + .. availability:: Windows + + .. versionadded:: next + + .. function:: cast(obj, type) This function is similar to the cast operator in C. It returns a new instance diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index e2926f2440af6d..f8f4188d27b472 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -60,6 +60,8 @@ interpreter. The :option:`-P ` command-line option and the ``show_positions`` argument were added. + The :option:`-S ` command-line option is added. + Example: Given the function :func:`!myfunc`:: def myfunc(alist): @@ -89,7 +91,7 @@ The :mod:`dis` module can be invoked as a script from the command line: .. code-block:: sh - python -m dis [-h] [-C] [-O] [-P] [infile] + python -m dis [-h] [-C] [-O] [-P] [-S] [infile] The following options are accepted: @@ -111,6 +113,10 @@ The following options are accepted: Show positions of instructions in the source code. +.. cmdoption:: -S, --specialized + + Show specialized bytecode. + If :file:`infile` is specified, its disassembled code will be written to stdout. Otherwise, disassembly is performed on compiled source code received from stdin. diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index 6b0282eed49566..106b0a6c95b7be 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -136,6 +136,10 @@ examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file :file:`Lib/test/test_doctest/test_doctest.py`. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + .. _doctest-simple-testmod: diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst index 4983b8961b1c3f..824d489818fac9 100644 --- a/Doc/library/errno.rst +++ b/Doc/library/errno.rst @@ -613,6 +613,13 @@ defined by the module. The specific list of defined symbols is available as No route to host +.. data:: EHWPOISON + + Memory page has hardware error. + + .. versionadded:: next + + .. data:: EALREADY Operation already in progress. This error is mapped to the diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index b5ba86f1b19223..f72b11e34c5c3d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -644,10 +644,16 @@ The following exceptions are the exceptions that are usually raised. The first index of invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. attribute:: end The index after the last invalid data in :attr:`object`. + This value should not be negative as it is interpreted as an + absolute offset but this constraint is not enforced at runtime. + .. exception:: UnicodeEncodeError diff --git a/Doc/library/io.rst b/Doc/library/io.rst index f793d7a7ef9a84..0d8cc5171d5476 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -64,6 +64,12 @@ In-memory text streams are also available as :class:`StringIO` objects:: f = io.StringIO("some initial text data") +.. note:: + + When working with a non-blocking stream, be aware that read operations on text I/O objects + might raise a :exc:`BlockingIOError` if the stream cannot perform the operation + immediately. + The text stream API is described in detail in the documentation of :class:`TextIOBase`. @@ -770,6 +776,11 @@ than raw I/O does. Read and return *size* bytes, or if *size* is not given or negative, until EOF or if the read call would block in non-blocking mode. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + .. method:: read1(size=-1, /) Read and return up to *size* bytes with only one call on the raw stream. @@ -779,6 +790,10 @@ than raw I/O does. .. versionchanged:: 3.7 The *size* argument is now optional. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) @@ -1007,6 +1022,11 @@ Text I/O .. versionchanged:: 3.10 The *encoding* argument now supports the ``"locale"`` dummy encoding name. + .. note:: + + When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` + may be raised if a read operation cannot be completed immediately. + :class:`TextIOWrapper` provides these data attributes and methods in addition to those from :class:`TextIOBase` and :class:`IOBase`: diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index c138e903fa5a0f..3b90d7830f3681 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -877,6 +877,11 @@ and :term:`generators ` which incur interpreter overhead. "Returns the sequence elements n times." return chain.from_iterable(repeat(tuple(iterable), n)) + def loops(n): + "Loop n times. Like range(n) but without creating integers." + # for _ in loops(100): ... + return repeat(None, n) + def tail(n, iterable): "Return an iterator over the last n items." # tail(3, 'ABCDEFG') → E F G @@ -1010,7 +1015,7 @@ The following recipes have a more mathematical flavor: .. testcode:: def powerset(iterable): - "powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" + # powerset([1,2,3]) → () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3) s = list(iterable) return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) @@ -1113,6 +1118,11 @@ The following recipes have a more mathematical flavor: if n > 1: yield n + def is_prime(n): + "Return True if n is prime." + # is_prime(1_000_000_000_000_403) → True + return n > 1 and next(factor(n)) == n + def totient(n): "Count of natural numbers up to n that are coprime to n." # https://mathworld.wolfram.com/TotientFunction.html @@ -1202,6 +1212,16 @@ The following recipes have a more mathematical flavor: [0, 2, 4, 6] + >>> for _ in loops(5): + ... print('hi') + ... + hi + hi + hi + hi + hi + + >>> list(tail(3, 'ABCDEFG')) ['E', 'F', 'G'] >>> # Verify the input is consumed greedily @@ -1475,6 +1495,23 @@ The following recipes have a more mathematical flavor: True + >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] + >>> list(filter(is_prime, range(-100, 100))) == small_primes + True + >>> carmichael = {561, 1105, 1729, 2465, 2821, 6601, 8911} # https://oeis.org/A002997 + >>> any(map(is_prime, carmichael)) + False + >>> # https://www.wolframalpha.com/input?i=is+128884753939+prime + >>> is_prime(128_884_753_939) # large prime + True + >>> is_prime(999953 * 999983) # large semiprime + False + >>> is_prime(1_000_000_000_000_007) # factor() example + False + >>> is_prime(1_000_000_000_000_403) # factor() example + True + + >>> list(factor(99)) # Code example 1 [3, 3, 11] >>> list(factor(1_000_000_000_000_007)) # Code example 2 diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index a42ac1f8bcdf71..4b48880d6d9a18 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -1289,35 +1289,6 @@ Reading directories raised. -.. method:: Path.scandir() - - When the path points to a directory, return an iterator of - :class:`os.DirEntry` objects corresponding to entries in the directory. The - returned iterator supports the :term:`context manager` protocol. It is - implemented using :func:`os.scandir` and gives the same guarantees. - - Using :meth:`~Path.scandir` instead of :meth:`~Path.iterdir` can - significantly increase the performance of code that also needs file type or - file attribute information, because :class:`os.DirEntry` objects expose - this information if the operating system provides it when scanning a - directory. - - The following example displays the names of subdirectories. The - ``entry.is_dir()`` check will generally not make an additional system call:: - - >>> p = Path('docs') - >>> with p.scandir() as entries: - ... for entry in entries: - ... if entry.is_dir(): - ... entry.name - ... - '_templates' - '_build' - '_static' - - .. versionadded:: 3.14 - - .. method:: Path.glob(pattern, *, case_sensitive=None, recurse_symlinks=False) Glob the given relative *pattern* in the directory represented by this path, diff --git a/Doc/library/select.rst b/Doc/library/select.rst index f23a249f44b485..4fcff9198944a8 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -317,11 +317,17 @@ Edge and Level Trigger Polling (epoll) Objects +-------------------------+-----------------------------------------------+ | :const:`EPOLLMSG` | Ignored. | +-------------------------+-----------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+-----------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 or later. + .. versionadded:: next + :const:`EPOLLWAKEUP` was added. It's only supported by Linux Kernel 3.5 + or later. + .. method:: epoll.close() Close the control file descriptor of the epoll object. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4f4fc9fba63120..d5f7714ced6873 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4149,6 +4149,21 @@ copying. .. versionchanged:: 3.5 The source format is no longer restricted when casting to a byte view. + .. method:: count(value, /) + + Count the number of occurrences of *value*. + + .. versionadded:: next + + .. method:: index(value, start=0, stop=sys.maxsize, /) + + Return the index of the first occurrence of *value* (at or after + index *start* and before index *stop*). + + Raises a :exc:`ValueError` if *value* cannot be found. + + .. versionadded:: next + There are also several readonly attributes available: .. attribute:: obj diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 04d28aee0f8672..b5b6e442e218fd 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -192,6 +192,10 @@ top-level directory where Python was built. On Windows, executing :program:`rt.bat` from your :file:`PCbuild` directory will run all regression tests. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + :mod:`test.support` --- Utilities for the Python test suite =========================================================== diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index d4b343db36efb3..f183f3f535c4cb 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -434,6 +434,18 @@ since it is impossible to detect the termination of alien threads. Multiple threads may be given the same name. The initial name is set by the constructor. + On some platforms, the thread name is set at the operating system level + when the thread starts, so that it is visible in task managers. + This name may be truncated to fit in a system-specific limit (for example, + 15 bytes on Linux or 63 bytes on macOS). + + Changes to *name* are only reflected at the OS level when the currently + running thread is renamed. (Setting the *name* attribute of a + different thread only updates the Python Thread object.) + + .. versionchanged:: 3.14 + Set the operating system thread name. + .. method:: getName() setName() diff --git a/Doc/library/token.rst b/Doc/library/token.rst index 0cc9dddd91ed6b..40982f32b4beee 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -79,6 +79,13 @@ the :mod:`tokenize` module. ``type_comments=True``. +.. data:: EXACT_TOKEN_TYPES + + A dictionary mapping the string representation of a token to its numeric code. + + .. versionadded:: 3.8 + + .. versionchanged:: 3.5 Added :data:`!AWAIT` and :data:`!ASYNC` tokens. diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 100a92b73d5497..4899ed64ebad8d 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -44,6 +44,10 @@ The module's API can be divided into two parts: necessary for later formatting without holding references to actual exception and traceback objects. +.. versionadded:: 3.13 + Output is colorized by default and can be + :ref:`controlled using environment variables `. + Module-Level Functions ---------------------- @@ -157,6 +161,13 @@ Module-Level Functions arguments have the same meaning as for :func:`print_stack`. +.. function:: print_list(extracted_list, file=None) + + Print the list of tuples as returned by :func:`extract_tb` or + :func:`extract_stack` as a formatted stack trace to the given file. + If *file* is ``None``, the output is written to :data:`sys.stderr`. + + .. function:: format_list(extracted_list) Given a list of tuples or :class:`FrameSummary` objects as returned by diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 38bad9405597dd..7f8b710f611002 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -46,7 +46,6 @@ test runner a textual interface, or return a special value to indicate the results of executing the tests. - .. seealso:: Module :mod:`doctest` @@ -198,6 +197,9 @@ For a list of all the command-line options:: In earlier versions it was only possible to run individual test methods and not modules or classes. +.. versionadded:: 3.14 + Output is colorized by default and can be + :ref:`controlled using environment variables `. Command-line options ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst index c57f433e6efd98..971e65605841e7 100644 --- a/Doc/library/xmlrpc.client.rst +++ b/Doc/library/xmlrpc.client.rst @@ -64,11 +64,11 @@ between conformable Python objects and XML on the wire. The obsolete *use_datetime* flag is similar to *use_builtin_types* but it applies only to date/time values. -.. versionchanged:: 3.3 - The *use_builtin_types* flag was added. + .. versionchanged:: 3.3 + The *use_builtin_types* flag was added. -.. versionchanged:: 3.8 - The *headers* parameter was added. + .. versionchanged:: 3.8 + The *headers* parameter was added. Both the HTTP and HTTPS transports support the URL syntax extension for HTTP Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` diff --git a/Doc/library/xmlrpc.rst b/Doc/library/xmlrpc.rst index 5f0a2cf68d01f9..a93d08f78cfba7 100644 --- a/Doc/library/xmlrpc.rst +++ b/Doc/library/xmlrpc.rst @@ -1,6 +1,9 @@ :mod:`!xmlrpc` --- XMLRPC server and client modules =================================================== +.. module:: xmlrpc + :synopsis: Server and client modules implementing XML-RPC. + XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a transport. With it, a client can call methods with parameters on a remote server (the server is named by a URI) and get back structured data. diff --git a/Doc/tools/.nitignore b/Doc/tools/.nitignore index 711c0b64095bd2..7d50aec56a9bf7 100644 --- a/Doc/tools/.nitignore +++ b/Doc/tools/.nitignore @@ -71,13 +71,11 @@ Doc/library/xmlrpc.server.rst Doc/library/zlib.rst Doc/reference/compound_stmts.rst Doc/reference/datamodel.rst -Doc/tutorial/datastructures.rst Doc/using/windows.rst Doc/whatsnew/2.4.rst Doc/whatsnew/2.5.rst Doc/whatsnew/2.6.rst Doc/whatsnew/2.7.rst -Doc/whatsnew/3.0.rst Doc/whatsnew/3.3.rst Doc/whatsnew/3.4.rst Doc/whatsnew/3.5.rst diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index 31941bc112a135..cbe780e075baf5 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -142,8 +142,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the -top of the stack, use :meth:`~list.pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`!append`. To retrieve an item from the +top of the stack, use :meth:`!pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +340,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method +value: the :keyword:`del` statement. This differs from the :meth:`!pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -500,8 +500,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`~list.append` and -:meth:`~list.extend`. +assignments, slice assignments, or methods like :meth:`!append` and +:meth:`!extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 6cf42b27718022..7db2f4820f346a 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -663,14 +663,6 @@ output. To control the color output only in the Python interpreter, the precedence over ``NO_COLOR``, which in turn takes precedence over ``FORCE_COLOR``. -.. Apparently this how you hack together a formatted link: - -.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` -.. _FORCE_COLOR: https://force-color.org/ - -.. |NO_COLOR| replace:: ``NO_COLOR`` -.. _NO_COLOR: https://no-color.org/ - Options you shouldn't use ~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 4d4eb2031ee980..aa43f75ec35a6c 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -292,10 +292,12 @@ To add Python to an iOS Xcode project: 10. Add Objective C code to initialize and use a Python interpreter in embedded mode. You should ensure that: - * :c:member:`UTF-8 mode ` is *enabled*; - * :c:member:`Buffered stdio ` is *disabled*; - * :c:member:`Writing bytecode ` is *disabled*; - * :c:member:`Signal handlers ` are *enabled*; + * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; + * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; + * Writing bytecode (:c:member:`PyConfig.write_bytecode`) is *disabled*; + * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; + * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* + (optional, but strongly recommended); * ``PYTHONHOME`` for the interpreter is configured to point at the ``python`` subfolder of your app's bundle; and * The ``PYTHONPATH`` for the interpreter includes: @@ -324,6 +326,49 @@ modules in your app, some additional steps will be required: * If you're using a separate folder for third-party packages, ensure that folder is included as part of the ``PYTHONPATH`` configuration in step 10. +Testing a Python package +------------------------ + +The CPython source tree contains :source:`a testbed project ` that +is used to run the CPython test suite on the iOS simulator. This testbed can also +be used as a testbed project for running your Python library's test suite on iOS. + +After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` +for details), create a clone of the Python iOS testbed project by running: + +.. code-block:: bash + + $ python iOS/testbed clone --framework --app --app app-testbed + +You will need to modify the ``iOS/testbed`` reference to point to that +directory in the CPython source tree; any folders specified with the ``--app`` +flag will be copied into the cloned testbed project. The resulting testbed will +be created in the ``app-testbed`` folder. In this example, the ``module1`` and +``module2`` would be importable modules at runtime. If your project has +additional dependencies, they can be installed into the +``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target +app-testbed/iOSTestbed/app_packages`` or similar). + +You can then use the ``app-testbed`` folder to run the test suite for your app, +For example, if ``module1.tests`` was the entry point to your test suite, you +could run: + +.. code-block:: bash + + $ python app-testbed run -- module1.tests + +This is the equivalent of running ``python -m module1.tests`` on a desktop +Python build. Any arguments after the ``--`` will be passed to the testbed as +if they were arguments to ``python -m`` on a desktop machine. + +You can also open the testbed project in Xcode by running: + +.. code-block:: bash + + $ open app-testbed/iOSTestbed.xcodeproj + +This will allow you to use the full Xcode suite of tools for debugging. + App Store Compliance ==================== diff --git a/Doc/whatsnew/3.0.rst b/Doc/whatsnew/3.0.rst index d97f5fdd9eaa4a..6e1fda22ed203a 100644 --- a/Doc/whatsnew/3.0.rst +++ b/Doc/whatsnew/3.0.rst @@ -150,8 +150,8 @@ Some well-known APIs no longer return lists: sorted(d)`` instead (this works in Python 2.5 too and is just as efficient). -* Also, the :meth:`dict.iterkeys`, :meth:`dict.iteritems` and - :meth:`dict.itervalues` methods are no longer supported. +* Also, the :meth:`!dict.iterkeys`, :meth:`!dict.iteritems` and + :meth:`!dict.itervalues` methods are no longer supported. * :func:`map` and :func:`filter` return iterators. If you really need a list and the input sequences are all of equal length, a quick @@ -170,7 +170,7 @@ Some well-known APIs no longer return lists: :func:`itertools.zip_longest`, e.g. ``map(func, *sequences)`` becomes ``list(map(func, itertools.zip_longest(*sequences)))``. -* :func:`range` now behaves like :func:`xrange` used to behave, except +* :func:`range` now behaves like :func:`!xrange` used to behave, except it works with values of arbitrary size. The latter no longer exists. @@ -192,33 +192,33 @@ Python 3.0 has simplified the rules for ordering comparisons: operators: objects of different incomparable types always compare unequal to each other. -* :meth:`builtin.sorted` and :meth:`list.sort` no longer accept the +* :meth:`sorted` and :meth:`list.sort` no longer accept the *cmp* argument providing a comparison function. Use the *key* argument instead. N.B. the *key* and *reverse* arguments are now "keyword-only". -* The :func:`cmp` function should be treated as gone, and the :meth:`__cmp__` - special method is no longer supported. Use :meth:`__lt__` for sorting, - :meth:`__eq__` with :meth:`__hash__`, and other rich comparisons as needed. - (If you really need the :func:`cmp` functionality, you could use the +* The :func:`!cmp` function should be treated as gone, and the :meth:`!__cmp__` + special method is no longer supported. Use :meth:`~object.__lt__` for sorting, + :meth:`~object.__eq__` with :meth:`~object.__hash__`, and other rich comparisons as needed. + (If you really need the :func:`!cmp` functionality, you could use the expression ``(a > b) - (a < b)`` as the equivalent for ``cmp(a, b)``.) Integers -------- -* :pep:`237`: Essentially, :class:`long` renamed to :class:`int`. +* :pep:`237`: Essentially, :class:`!long` renamed to :class:`int`. That is, there is only one built-in integral type, named - :class:`int`; but it behaves mostly like the old :class:`long` type. + :class:`int`; but it behaves mostly like the old :class:`!long` type. * :pep:`238`: An expression like ``1/2`` returns a float. Use ``1//2`` to get the truncating behavior. (The latter syntax has existed for years, at least since Python 2.2.) -* The :data:`sys.maxint` constant was removed, since there is no +* The :data:`!sys.maxint` constant was removed, since there is no longer a limit to the value of integers. However, :data:`sys.maxsize` can be used as an integer larger than any practical list or string index. It conforms to the implementation's "natural" integer size - and is typically the same as :data:`sys.maxint` in previous releases + and is typically the same as :data:`!sys.maxint` in previous releases on the same platform (assuming the same build options). * The :func:`repr` of a long integer doesn't include the trailing ``L`` @@ -251,7 +251,7 @@ changed. that uses Unicode, encodings or binary data most likely has to change. The change is for the better, as in the 2.x world there were numerous bugs having to do with mixing encoded and unencoded - text. To be prepared in Python 2.x, start using :class:`unicode` + text. To be prepared in Python 2.x, start using :class:`!unicode` for all unencoded text, and :class:`str` for binary or encoded data only. Then the ``2to3`` tool will do most of the work for you. @@ -269,7 +269,7 @@ changed. separate *mutable* type to hold buffered binary data, :class:`bytearray`. Nearly all APIs that accept :class:`bytes` also accept :class:`bytearray`. The mutable API is based on - :class:`collections.MutableSequence`. + :class:`collections.MutableSequence `. * All backslashes in raw string literals are interpreted literally. This means that ``'\U'`` and ``'\u'`` escapes in raw strings are not @@ -278,11 +278,11 @@ changed. single "euro" character. (Of course, this change only affects raw string literals; the euro character is ``'\u20ac'`` in Python 3.0.) -* The built-in :class:`basestring` abstract type was removed. Use +* The built-in :class:`!basestring` abstract type was removed. Use :class:`str` instead. The :class:`str` and :class:`bytes` types don't have functionality enough in common to warrant a shared base class. The ``2to3`` tool (see below) replaces every occurrence of - :class:`basestring` with :class:`str`. + :class:`!basestring` with :class:`str`. * Files opened as text files (still the default mode for :func:`open`) always use an encoding to map between strings (in memory) and bytes @@ -428,7 +428,7 @@ Changed Syntax class C(metaclass=M): ... - The module-global :data:`__metaclass__` variable is no longer + The module-global :data:`!__metaclass__` variable is no longer supported. (It was a crutch to make it easier to default to new-style classes without deriving every class from :class:`object`.) @@ -522,19 +522,19 @@ consulted for longer descriptions. *encoding*, *errors*, *newline* and *closefd*. Also note that an invalid *mode* argument now raises :exc:`ValueError`, not :exc:`IOError`. The binary file object underlying a text file - object can be accessed as :attr:`f.buffer` (but beware that the + object can be accessed as :attr:`!f.buffer` (but beware that the text object maintains a buffer of itself in order to speed up the encoding and decoding operations). -* :ref:`pep-3118`. The old builtin :func:`buffer` is now really gone; +* :ref:`pep-3118`. The old builtin :func:`!buffer` is now really gone; the new builtin :func:`memoryview` provides (mostly) similar functionality. * :ref:`pep-3119`. The :mod:`abc` module and the ABCs defined in the :mod:`collections` module plays a somewhat more prominent role in the language now, and built-in collection types like :class:`dict` - and :class:`list` conform to the :class:`collections.MutableMapping` - and :class:`collections.MutableSequence` ABCs, respectively. + and :class:`list` conform to the :class:`collections.MutableMapping ` + and :class:`collections.MutableSequence ` ABCs, respectively. * :ref:`pep-3127`. As mentioned above, the new octal literal notation is the only one supported, and binary literals have been @@ -612,7 +612,7 @@ review: :mod:`!CGIHTTPServer`, :mod:`!SimpleHTTPServer`, :mod:`!Cookie`, :mod:`!cookielib`). - * :mod:`tkinter` (all :mod:`Tkinter`-related modules except + * :mod:`tkinter` (all ``Tkinter``-related modules except :mod:`turtle`). The target audience of :mod:`turtle` doesn't really care about :mod:`tkinter`. Also note that as of Python 2.6, the functionality of :mod:`turtle` has been greatly enhanced. @@ -628,47 +628,47 @@ Some other changes to standard library modules, not covered by * Killed :mod:`!sets`. Use the built-in :func:`set` class. -* Cleanup of the :mod:`sys` module: removed :func:`sys.exitfunc`, - :func:`sys.exc_clear`, :data:`sys.exc_type`, :data:`sys.exc_value`, - :data:`sys.exc_traceback`. (Note that :data:`sys.last_type` +* Cleanup of the :mod:`sys` module: removed :func:`!sys.exitfunc`, + :func:`!sys.exc_clear`, :data:`!sys.exc_type`, :data:`!sys.exc_value`, + :data:`!sys.exc_traceback`. (Note that :data:`sys.last_type` etc. remain.) -* Cleanup of the :class:`array.array` type: the :meth:`read` and - :meth:`write` methods are gone; use :meth:`fromfile` and - :meth:`tofile` instead. Also, the ``'c'`` typecode for array is +* Cleanup of the :class:`array.array` type: the :meth:`!read` and + :meth:`!write` methods are gone; use :meth:`~array.array.fromfile` and + :meth:`~array.array.tofile` instead. Also, the ``'c'`` typecode for array is gone -- use either ``'b'`` for bytes or ``'u'`` for Unicode characters. * Cleanup of the :mod:`operator` module: removed - :func:`sequenceIncludes` and :func:`isCallable`. + :func:`!sequenceIncludes` and :func:`!isCallable`. * Cleanup of the :mod:`!thread` module: :func:`!acquire_lock` and :func:`!release_lock` are gone; use :meth:`~threading.Lock.acquire` and :meth:`~threading.Lock.release` instead. -* Cleanup of the :mod:`random` module: removed the :func:`jumpahead` API. +* Cleanup of the :mod:`random` module: removed the :func:`!jumpahead` API. * The :mod:`!new` module is gone. -* The functions :func:`os.tmpnam`, :func:`os.tempnam` and - :func:`os.tmpfile` have been removed in favor of the :mod:`tempfile` +* The functions :func:`!os.tmpnam`, :func:`!os.tempnam` and + :func:`!os.tmpfile` have been removed in favor of the :mod:`tempfile` module. * The :mod:`tokenize` module has been changed to work with bytes. The main entry point is now :func:`tokenize.tokenize`, instead of generate_tokens. -* :data:`string.letters` and its friends (:data:`string.lowercase` and - :data:`string.uppercase`) are gone. Use +* :data:`!string.letters` and its friends (:data:`!string.lowercase` and + :data:`!string.uppercase`) are gone. Use :data:`string.ascii_letters` etc. instead. (The reason for the - removal is that :data:`string.letters` and friends had + removal is that :data:`!string.letters` and friends had locale-specific behavior, which is a bad idea for such attractively named global "constants".) -* Renamed module :mod:`__builtin__` to :mod:`builtins` (removing the - underscores, adding an 's'). The :data:`__builtins__` variable +* Renamed module :mod:`!__builtin__` to :mod:`builtins` (removing the + underscores, adding an 's'). The :data:`!__builtins__` variable found in most global namespaces is unchanged. To modify a builtin, - you should use :mod:`builtins`, not :data:`__builtins__`! + you should use :mod:`builtins`, not :data:`!__builtins__`! :pep:`3101`: A New Approach To String Formatting @@ -702,9 +702,9 @@ new powerful features added: idiom for handling all exceptions except for this latter category is to use :keyword:`except` :exc:`Exception`. -* :exc:`StandardError` was removed. +* :exc:`!StandardError` was removed. -* Exceptions no longer behave as sequences. Use the :attr:`args` +* Exceptions no longer behave as sequences. Use the :attr:`~BaseException.args` attribute instead. * :pep:`3109`: Raising exceptions. You must now use :samp:`raise @@ -765,20 +765,20 @@ Operators And Special Methods When referencing a method as a class attribute, you now get a plain function object. -* :meth:`__getslice__`, :meth:`__setslice__` and :meth:`__delslice__` +* :meth:`!__getslice__`, :meth:`!__setslice__` and :meth:`!__delslice__` were killed. The syntax ``a[i:j]`` now translates to - ``a.__getitem__(slice(i, j))`` (or :meth:`__setitem__` or - :meth:`__delitem__`, when used as an assignment or deletion target, + ``a.__getitem__(slice(i, j))`` (or :meth:`~object.__setitem__` or + :meth:`~object.__delitem__`, when used as an assignment or deletion target, respectively). * :pep:`3114`: the standard :meth:`next` method has been renamed to :meth:`~iterator.__next__`. -* The :meth:`__oct__` and :meth:`__hex__` special methods are removed - -- :func:`oct` and :func:`hex` use :meth:`__index__` now to convert +* The :meth:`!__oct__` and :meth:`!__hex__` special methods are removed + -- :func:`oct` and :func:`hex` use :meth:`~object.__index__` now to convert the argument to an integer. -* Removed support for :attr:`__members__` and :attr:`__methods__`. +* Removed support for :attr:`!__members__` and :attr:`!__methods__`. * The function attributes named :attr:`!func_X` have been renamed to use the :attr:`!__X__` form, freeing up these names in the function @@ -802,7 +802,7 @@ Builtins instance will automatically be chosen. With arguments, the behavior of :func:`super` is unchanged. -* :pep:`3111`: :func:`raw_input` was renamed to :func:`input`. That +* :pep:`3111`: :func:`!raw_input` was renamed to :func:`input`. That is, the new :func:`input` function reads a line from :data:`sys.stdin` and returns it with the trailing newline stripped. It raises :exc:`EOFError` if the input is terminated prematurely. @@ -820,31 +820,31 @@ Builtins argument and a value of the same type as ``x`` when called with two arguments. -* Moved :func:`intern` to :func:`sys.intern`. +* Moved :func:`!intern` to :func:`sys.intern`. -* Removed: :func:`apply`. Instead of ``apply(f, args)`` use +* Removed: :func:`!apply`. Instead of ``apply(f, args)`` use ``f(*args)``. * Removed :func:`callable`. Instead of ``callable(f)`` you can use - ``isinstance(f, collections.Callable)``. The :func:`operator.isCallable` + ``isinstance(f, collections.Callable)``. The :func:`!operator.isCallable` function is also gone. -* Removed :func:`coerce`. This function no longer serves a purpose +* Removed :func:`!coerce`. This function no longer serves a purpose now that classic classes are gone. -* Removed :func:`execfile`. Instead of ``execfile(fn)`` use +* Removed :func:`!execfile`. Instead of ``execfile(fn)`` use ``exec(open(fn).read())``. -* Removed the :class:`file` type. Use :func:`open`. There are now several +* Removed the :class:`!file` type. Use :func:`open`. There are now several different kinds of streams that open can return in the :mod:`io` module. -* Removed :func:`reduce`. Use :func:`functools.reduce` if you really +* Removed :func:`!reduce`. Use :func:`functools.reduce` if you really need it; however, 99 percent of the time an explicit :keyword:`for` loop is more readable. -* Removed :func:`reload`. Use :func:`!imp.reload`. +* Removed :func:`!reload`. Use :func:`!imp.reload`. -* Removed. :meth:`dict.has_key` -- use the :keyword:`in` operator +* Removed. :meth:`!dict.has_key` -- use the :keyword:`in` operator instead. .. ====================================================================== diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index 664b1866172378..9f6d98b9950d19 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -252,15 +252,6 @@ Improved error messages the canonical |NO_COLOR|_ and |FORCE_COLOR|_ environment variables. (Contributed by Pablo Galindo Salgado in :gh:`112730`.) -.. Apparently this how you hack together a formatted link: - (https://www.docutils.org/docs/ref/rst/directives.html#replacement-text) - -.. |FORCE_COLOR| replace:: ``FORCE_COLOR`` -.. _FORCE_COLOR: https://force-color.org/ - -.. |NO_COLOR| replace:: ``NO_COLOR`` -.. _NO_COLOR: https://no-color.org/ - * A common mistake is to write a script with the same name as a standard library module. When this results in errors, we now display a more helpful error message: diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 869a47c1261293..b71d31f9742fe0 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -245,6 +245,13 @@ Other language changes making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`126012`.) +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in + :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) New modules =========== @@ -313,9 +320,12 @@ ctypes to help match a non-default ABI. (Contributed by Petr Viktorin in :gh:`97702`.) -* The :exc:`~ctypes.COMError` exception is now public. +* On Windows, the :exc:`~ctypes.COMError` exception is now public. (Contributed by Jun Komoda in :gh:`126686`.) +* On Windows, the :func:`~ctypes.CopyComPointer` function is now public. + (Contributed by Jun Komoda in :gh:`127275`.) + datetime -------- @@ -345,6 +355,17 @@ dis This feature is also exposed via :option:`dis --show-positions`. (Contributed by Bénédikt Tran in :gh:`123165`.) +* Add the :option:`dis --specialized` command-line option to + show specialized bytecode. + (Contributed by Bénédikt Tran in :gh:`127413`.) + + +errno +----- + +* Add :data:`errno.EHWPOISON` error code. + (Contributed by James Roy in :gh:`126585`.) + fractions --------- @@ -404,6 +425,15 @@ inspect (Contributed by Zhikang Yan in :gh:`125634`.) + +io +-- + +* Reading text from a non-blocking stream with ``read`` may now raise a + :exc:`BlockingIOError` if the operation cannot immediately return bytes. + (Contributed by Giovanni Siragusa in :gh:`109523`.) + + json ---- @@ -509,12 +539,6 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :meth:`pathlib.Path.scandir` to scan a directory and return an iterator - of :class:`os.DirEntry` objects. This is exactly equivalent to calling - :func:`os.scandir` on a path object. - - (Contributed by Barney Gale in :gh:`125413`.) - pdb --- @@ -576,6 +600,14 @@ sys from other interpreters than the one it's called in. +tkinter +------- + +* Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept + arguments passed by keyword. + (Contributed by Zhikang Yan in :gh:`126899`.) + + unicodedata ----------- @@ -585,6 +617,13 @@ unicodedata unittest -------- +* :mod:`unittest` output is now colored by default. + This can be controlled via the :envvar:`PYTHON_COLORS` environment + variable as well as the canonical |NO_COLOR|_ + and |FORCE_COLOR|_ environment variables. + See also :ref:`using-on-controlling-color`. + (Contributed by Hugo van Kemenade in :gh:`127221`.) + * unittest discovery supports :term:`namespace package` as start directory again. It was removed in Python 3.11. (Contributed by Jacob Walls in :gh:`80958`.) diff --git a/Include/Python.h b/Include/Python.h index 717e27feab62db..64be80145890a3 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -69,6 +69,7 @@ #include "pystats.h" #include "pyatomic.h" #include "lock.h" +#include "critical_section.h" #include "object.h" #include "refcount.h" #include "objimpl.h" @@ -130,7 +131,6 @@ #include "import.h" #include "abstract.h" #include "bltinmodule.h" -#include "critical_section.h" #include "cpython/pyctype.h" #include "pystrtod.h" #include "pystrcmp.h" diff --git a/Include/cpython/cellobject.h b/Include/cpython/cellobject.h index 47a6a491497ea0..85a63a13747d87 100644 --- a/Include/cpython/cellobject.h +++ b/Include/cpython/cellobject.h @@ -22,10 +22,14 @@ PyAPI_FUNC(PyObject *) PyCell_Get(PyObject *); PyAPI_FUNC(int) PyCell_Set(PyObject *, PyObject *); static inline PyObject* PyCell_GET(PyObject *op) { + PyObject *res; PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); - return cell->ob_ref; + Py_BEGIN_CRITICAL_SECTION(cell); + res = cell->ob_ref; + Py_END_CRITICAL_SECTION(); + return res; } #define PyCell_GET(op) PyCell_GET(_PyObject_CAST(op)) @@ -33,7 +37,9 @@ static inline void PyCell_SET(PyObject *op, PyObject *value) { PyCellObject *cell; assert(PyCell_Check(op)); cell = _Py_CAST(PyCellObject*, op); + Py_BEGIN_CRITICAL_SECTION(cell); cell->ob_ref = value; + Py_END_CRITICAL_SECTION(); } #define PyCell_SET(op, value) PyCell_SET(_PyObject_CAST(op), (value)) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index f69c586a4f96f3..8ef19f677066c2 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -179,6 +179,9 @@ typedef struct PyConfig { int use_frozen_modules; int safe_path; int int_max_str_digits; +#ifdef __APPLE__ + int use_system_logger; +#endif int cpu_count; #ifdef Py_GIL_DISABLED diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 4ecef4f56edf42..6d106c1b499c69 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -321,6 +321,27 @@ _Py_atomic_load_ptr(const void *obj); static inline int _Py_atomic_load_int_relaxed(const int *obj); +static inline char +_Py_atomic_load_char_relaxed(const char *obj); + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj); + +static inline short +_Py_atomic_load_short_relaxed(const short *obj); + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj); + +static inline long +_Py_atomic_load_long_relaxed(const long *obj); + +static inline double +_Py_atomic_load_double_relaxed(const double *obj); + +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj); + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj); @@ -458,6 +479,30 @@ static inline void _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value); +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value); + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value); + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value); + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value); + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value); + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value); + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value); + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value); + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index ef09954d53ac1d..b179e4c9a185a9 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -306,6 +306,34 @@ static inline int _Py_atomic_load_int_relaxed(const int *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ float ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ double ret; __atomic_load(obj, &ret, __ATOMIC_RELAXED); return ret; } + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } @@ -362,6 +390,10 @@ static inline unsigned long long _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) { return __atomic_load_n(obj, __ATOMIC_RELAXED); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ return __atomic_load_n(obj, __ATOMIC_RELAXED); } + // --- _Py_atomic_store ------------------------------------------------------ @@ -485,6 +517,38 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, unsigned long long value) { __atomic_store_n(obj, value, __ATOMIC_RELAXED); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ __atomic_store(obj, &value, __ATOMIC_RELAXED); } + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ __atomic_store_n(obj, value, __ATOMIC_RELAXED); } + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index 84da21bdcbff4f..d155955df0cddf 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -634,6 +634,48 @@ _Py_atomic_load_int_relaxed(const int *obj) return *(volatile int *)obj; } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + return *(volatile char *)obj; +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + return *(volatile unsigned char *)obj; +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + return *(volatile short *)obj; +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + return *(volatile unsigned short *)obj; +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + return *(volatile long *)obj; +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + return *(volatile float *)obj; +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + return *(volatile double *)obj; +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -718,6 +760,12 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) return *(volatile unsigned long long *)obj; } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + return *(volatile long long *)obj; +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -899,6 +947,60 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, *(volatile unsigned long long *)obj = value; } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + *(volatile char *)obj = value; +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + *(volatile unsigned char *)obj = value; +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + *(volatile short *)obj = value; +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + *(volatile unsigned short *)obj = value; +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + *(volatile unsigned int *)obj = value; +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + *(volatile long *)obj = value; +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + *(volatile float *)obj = value; +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + *(volatile double *)obj = value; +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + *(volatile long long *)obj = value; +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 7c71e94c68f8e6..69a8b9e615ea5f 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -515,6 +515,62 @@ _Py_atomic_load_int_relaxed(const int *obj) memory_order_relaxed); } +static inline char +_Py_atomic_load_char_relaxed(const char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(char)*)obj, + memory_order_relaxed); +} + +static inline unsigned char +_Py_atomic_load_uchar_relaxed(const unsigned char *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned char)*)obj, + memory_order_relaxed); +} + +static inline short +_Py_atomic_load_short_relaxed(const short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(short)*)obj, + memory_order_relaxed); +} + +static inline unsigned short +_Py_atomic_load_ushort_relaxed(const unsigned short *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(unsigned short)*)obj, + memory_order_relaxed); +} + +static inline long +_Py_atomic_load_long_relaxed(const long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long)*)obj, + memory_order_relaxed); +} + +static inline float +_Py_atomic_load_float_relaxed(const float *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(float)*)obj, + memory_order_relaxed); +} + +static inline double +_Py_atomic_load_double_relaxed(const double *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(double)*)obj, + memory_order_relaxed); +} + static inline int8_t _Py_atomic_load_int8_relaxed(const int8_t *obj) { @@ -627,6 +683,14 @@ _Py_atomic_load_ullong_relaxed(const unsigned long long *obj) memory_order_relaxed); } +static inline long long +_Py_atomic_load_llong_relaxed(const long long *obj) +{ + _Py_USING_STD; + return atomic_load_explicit((const _Atomic(long long)*)obj, + memory_order_relaxed); +} + // --- _Py_atomic_store ------------------------------------------------------ @@ -852,6 +916,78 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj, memory_order_relaxed); } +static inline void +_Py_atomic_store_char_relaxed(char *obj, char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uchar_relaxed(unsigned char *obj, unsigned char value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned char)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_short_relaxed(short *obj, short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned short)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_long_relaxed(long *obj, long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_float_relaxed(float *obj, float value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(float)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_double_relaxed(double *obj, double value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(double)*)obj, value, + memory_order_relaxed); +} + +static inline void +_Py_atomic_store_llong_relaxed(long long *obj, long long value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(long long)*)obj, value, + memory_order_relaxed); +} + // --- _Py_atomic_load_ptr_acquire / _Py_atomic_store_ptr_release ------------ diff --git a/Include/cpython/pystats.h b/Include/cpython/pystats.h index f1ca54839fbc38..2ae48002d720e9 100644 --- a/Include/cpython/pystats.h +++ b/Include/cpython/pystats.h @@ -99,6 +99,8 @@ typedef struct _gc_stats { uint64_t collections; uint64_t object_visits; uint64_t objects_collected; + uint64_t objects_transitively_reachable; + uint64_t objects_not_transitively_reachable; } GCStats; typedef struct _uop_stats { diff --git a/Include/cpython/tracemalloc.h b/Include/cpython/tracemalloc.h index 61a16ea9a9f3eb..6d094291ae2e90 100644 --- a/Include/cpython/tracemalloc.h +++ b/Include/cpython/tracemalloc.h @@ -1,6 +1,9 @@ #ifndef Py_LIMITED_API #ifndef Py_TRACEMALLOC_H #define Py_TRACEMALLOC_H +#ifdef __cplusplus +extern "C" { +#endif /* Track an allocated memory block in the tracemalloc module. Return 0 on success, return -1 on error (failed to allocate memory to store @@ -22,5 +25,8 @@ PyAPI_FUNC(int) PyTraceMalloc_Untrack( unsigned int domain, uintptr_t ptr); +#ifdef __cplusplus +} +#endif #endif // !Py_TRACEMALLOC_H #endif // !Py_LIMITED_API diff --git a/Include/internal/pycore_atexit.h b/Include/internal/pycore_atexit.h index 507a5c03cbc792..cde5b530baf00c 100644 --- a/Include/internal/pycore_atexit.h +++ b/Include/internal/pycore_atexit.h @@ -44,7 +44,6 @@ typedef struct { struct atexit_state { atexit_callback *ll_callbacks; - atexit_callback *last_ll_callback; // XXX The rest of the state could be moved to the atexit module state // and a low-level callback added for it during module exec. diff --git a/Include/internal/pycore_backoff.h b/Include/internal/pycore_backoff.h index 3e02728522828e..b5e33fa8b7abc0 100644 --- a/Include/internal/pycore_backoff.h +++ b/Include/internal/pycore_backoff.h @@ -115,10 +115,9 @@ initial_jump_backoff_counter(void) /* Initial exit temperature. * Must be larger than ADAPTIVE_COOLDOWN_VALUE, * otherwise when a side exit warms up we may construct - * a new trace before the Tier 1 code has properly re-specialized. - * Backoff sequence 64, 128, 256, 512, 1024, 2048, 4096. */ -#define SIDE_EXIT_INITIAL_VALUE 63 -#define SIDE_EXIT_INITIAL_BACKOFF 6 + * a new trace before the Tier 1 code has properly re-specialized. */ +#define SIDE_EXIT_INITIAL_VALUE 4095 +#define SIDE_EXIT_INITIAL_BACKOFF 12 static inline _Py_BackoffCounter initial_temperature_backoff_counter(void) diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index a0acf76db6f04d..d607a54aa4a2f5 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -609,12 +609,19 @@ PyAPI_DATA(const struct _PyCode8) _Py_InitCleanup; #ifdef Py_GIL_DISABLED +static inline _PyCodeArray * +_PyCode_GetTLBCArray(PyCodeObject *co) +{ + return _Py_STATIC_CAST(_PyCodeArray *, + _Py_atomic_load_ptr_acquire(&co->co_tlbc)); +} + // Return a pointer to the thread-local bytecode for the current thread, if it // exists. static inline _Py_CODEUNIT * _PyCode_GetTLBCFast(PyThreadState *tstate, PyCodeObject *co) { - _PyCodeArray *code = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + _PyCodeArray *code = _PyCode_GetTLBCArray(co); int32_t idx = ((_PyThreadStateImpl*) tstate)->tlbc_index; if (idx < code->size && code->entries[idx] != NULL) { return (_Py_CODEUNIT *) code->entries[idx]; diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index 8c0100390d036e..96ae4dd22ecb43 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -75,6 +75,7 @@ typedef struct _PyInterpreterFrame { _PyStackRef *stackpointer; uint16_t return_offset; /* Only relevant during a function call */ char owner; + char visited; /* Locals and stack */ _PyStackRef localsplus[1]; } _PyInterpreterFrame; @@ -93,7 +94,7 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) { #ifdef Py_GIL_DISABLED PyCodeObject *co = _PyFrame_GetCode(f); - _PyCodeArray *tlbc = _Py_atomic_load_ptr_acquire(&co->co_tlbc); + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); return (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; #else @@ -207,6 +208,7 @@ _PyFrame_Initialize( #endif frame->return_offset = 0; frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; for (int i = null_locals_from; i < code->co_nlocalsplus; i++) { frame->localsplus[i] = PyStackRef_NULL; @@ -389,6 +391,7 @@ _PyFrame_PushTrampolineUnchecked(PyThreadState *tstate, PyCodeObject *code, int frame->instr_ptr = _PyCode_CODE(code); #endif frame->owner = FRAME_OWNED_BY_THREAD; + frame->visited = 0; frame->return_offset = 0; #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_freelist.h b/Include/internal/pycore_freelist.h index da2d7bf6ae1393..84a5ab30f3eeea 100644 --- a/Include/internal/pycore_freelist.h +++ b/Include/internal/pycore_freelist.h @@ -51,7 +51,7 @@ static inline int _PyFreeList_Push(struct _Py_freelist *fl, void *obj, Py_ssize_t maxsize) { if (fl->size < maxsize && fl->size >= 0) { - *(void **)obj = fl->freelist; + FT_ATOMIC_STORE_PTR_RELAXED(*(void **)obj, fl->freelist); fl->freelist = obj; fl->size++; OBJECT_STAT_INC(to_freelist); diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index 479fe10d00066d..4ff34bf8ead7d0 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -10,11 +10,11 @@ extern "C" { /* GC information is stored BEFORE the object structure. */ typedef struct { - // Pointer to next object in the list. + // Tagged pointer to next object in the list. // 0 means the object is not tracked uintptr_t _gc_next; - // Pointer to previous object in the list. + // Tagged pointer to previous object in the list. // Lowest two bits are used for flags documented later. uintptr_t _gc_prev; } PyGC_Head; @@ -284,6 +284,11 @@ struct gc_generation_stats { Py_ssize_t uncollectable; }; +enum _GCPhase { + GC_PHASE_MARK = 0, + GC_PHASE_COLLECT = 1 +}; + struct _gc_runtime_state { /* List of objects that still need to be cleaned up, singly linked * via their gc headers' gc_prev pointers. */ @@ -311,6 +316,7 @@ struct _gc_runtime_state { Py_ssize_t work_to_do; /* Which of the old spaces is the visited space */ int visited_space; + int phase; #ifdef Py_GIL_DISABLED /* This is the number of objects that survived the last full diff --git a/Include/internal/pycore_list.h b/Include/internal/pycore_list.h index 2c666f9be4bd79..836ff30abfcedb 100644 --- a/Include/internal/pycore_list.h +++ b/Include/internal/pycore_list.h @@ -10,6 +10,9 @@ extern "C" { PyAPI_FUNC(PyObject*) _PyList_Extend(PyListObject *, PyObject *); extern void _PyList_DebugMallocStats(FILE *out); +// _PyList_GetItemRef should be used only when the object is known as a list +// because it doesn't raise TypeError when the object is not a list, whereas PyList_GetItemRef does. +extern PyObject* _PyList_GetItemRef(PyListObject *, Py_ssize_t i); #define _PyList_ITEMS(op) _Py_RVALUE(_PyList_CAST(op)->ob_item) @@ -59,7 +62,7 @@ typedef struct { union _PyStackRef; PyAPI_FUNC(PyObject *)_PyList_FromStackRefSteal(const union _PyStackRef *src, Py_ssize_t n); - +PyAPI_FUNC(PyObject *)_PyList_AsTupleAndClear(PyListObject *v); #ifdef __cplusplus } diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 34d835a7f84ee7..668ea47ca727e2 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -73,15 +73,25 @@ PyAPI_FUNC(int) _PyObject_IsFreed(PyObject *); #define _PyObject_HEAD_INIT(type) \ { \ .ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ .ob_type = (type) \ } #else +#if SIZEOF_VOID_P > 4 #define _PyObject_HEAD_INIT(type) \ { \ - .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ + .ob_refcnt = _Py_IMMORTAL_INITIAL_REFCNT, \ + .ob_flags = _Py_STATICALLY_ALLOCATED_FLAG, \ + .ob_type = (type) \ + } +#else +#define _PyObject_HEAD_INIT(type) \ + { \ + .ob_refcnt = _Py_STATIC_IMMORTAL_INITIAL_REFCNT, \ .ob_type = (type) \ } #endif +#endif #define _PyVarObject_HEAD_INIT(type, size) \ { \ .ob_base = _PyObject_HEAD_INIT(type), \ @@ -127,7 +137,11 @@ static inline void _Py_RefcntAdd(PyObject* op, Py_ssize_t n) _Py_AddRefTotal(_PyThreadState_GET(), n); #endif #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt += (PY_UINT32_T)n; +#else op->ob_refcnt += n; +#endif #else if (_Py_IsOwnedByCurrentThread(op)) { uint32_t local = op->ob_ref_local; @@ -170,7 +184,7 @@ PyAPI_FUNC(void) _Py_SetImmortalUntracked(PyObject *op); // Makes an immortal object mortal again with the specified refcnt. Should only // be used during runtime finalization. -static inline void _Py_SetMortal(PyObject *op, Py_ssize_t refcnt) +static inline void _Py_SetMortal(PyObject *op, short refcnt) { if (op) { assert(_Py_IsImmortal(op)); @@ -471,8 +485,8 @@ static inline void _PyObject_GC_TRACK( PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - /* Young objects will be moved into the visited space during GC, so set the bit here */ - gc->_gc_next = ((uintptr_t)generation0) | (uintptr_t)interp->gc.visited_space; + uintptr_t not_visited = 1 ^ interp->gc.visited_space; + gc->_gc_next = ((uintptr_t)generation0) | not_visited; generation0->_gc_prev = (uintptr_t)gc; #endif } @@ -697,8 +711,52 @@ _PyObject_SetMaybeWeakref(PyObject *op) } } +extern int _PyObject_ResurrectEndSlow(PyObject *op); #endif +// Temporarily resurrects an object during deallocation. The refcount is set +// to one. +static inline void +_PyObject_ResurrectStart(PyObject *op) +{ + assert(Py_REFCNT(op) == 0); +#ifdef Py_REF_DEBUG + _Py_IncRefTotal(_PyThreadState_GET()); +#endif +#ifdef Py_GIL_DISABLED + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_ThreadId()); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 1); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); +#else + Py_SET_REFCNT(op, 1); +#endif +} + +// Undoes an object resurrection by decrementing the refcount without calling +// _Py_Dealloc(). Returns 0 if the object is dead (the normal case), and +// deallocation should continue. Returns 1 if the object is still alive. +static inline int +_PyObject_ResurrectEnd(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_DecRefTotal(_PyThreadState_GET()); +#endif +#ifndef Py_GIL_DISABLED + Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + return Py_REFCNT(op) != 0; +#else + uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); + Py_ssize_t shared = _Py_atomic_load_ssize_acquire(&op->ob_ref_shared); + if (_Py_IsOwnedByCurrentThread(op) && local == 1 && shared == 0) { + // Fast-path: object has a single refcount and is owned by this thread + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, 0); + return 0; + } + // Slow-path: object has a shared refcount or is not owned by this thread + return _PyObject_ResurrectEndSlow(op); +#endif +} + /* Tries to incref op and returns 1 if successful or 0 otherwise. */ static inline int _Py_TryIncref(PyObject *op) @@ -835,6 +893,20 @@ extern int _PyObject_StoreInstanceAttribute(PyObject *obj, PyObject *name, PyObject *value); extern bool _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr); +extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, + unsigned int *); + +// Cache the provided init method in the specialization cache of type if the +// provided type version matches the current version of the type. +// +// The cached value is borrowed and is only valid if guarded by a type +// version check. In free-threaded builds the init method must also use +// deferred reference counting. +// +// Returns 1 if the value was cached or 0 otherwise. +extern int _PyType_CacheInitForSpecialization(PyHeapTypeObject *type, + PyObject *init, + unsigned int tp_version); #ifdef Py_GIL_DISABLED # define MANAGED_DICT_OFFSET (((Py_ssize_t)sizeof(PyObject *))*-1) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 5ce172856e1b19..81dde66a6f26c2 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1952,7 +1952,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, - [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, + [BINARY_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, [BINARY_SUBSCR_STR_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BINARY_SUBSCR_TUPLE_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [BUILD_LIST] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, @@ -2148,7 +2148,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[266] = { [UNARY_NOT] = { true, INSTR_FMT_IX, HAS_PURE_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index a1bb383bcd22e9..d755d03a5fa190 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -61,6 +61,54 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ + _Py_atomic_store_char_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ + _Py_atomic_load_char_relaxed(&value) +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) \ + _Py_atomic_store_uchar_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) \ + _Py_atomic_load_uchar_relaxed(&value) +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) \ + _Py_atomic_store_short_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) \ + _Py_atomic_load_short_relaxed(&value) +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) \ + _Py_atomic_store_ushort_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ + _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ + _Py_atomic_store_int_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_INT_RELAXED(value) \ + _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ + _Py_atomic_store_uint_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ + _Py_atomic_load_uint_relaxed(&value) +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) \ + _Py_atomic_store_long_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) \ + _Py_atomic_load_long_relaxed(&value) +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) \ + _Py_atomic_store_ulong_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ + _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ + _Py_atomic_store_float_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ + _Py_atomic_load_float_relaxed(&value) +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) \ + _Py_atomic_store_double_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) \ + _Py_atomic_load_double_relaxed(&value) +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) \ + _Py_atomic_store_llong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) \ + _Py_atomic_load_llong_relaxed(&value) +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) \ + _Py_atomic_store_ullong_relaxed(&value, new_value) +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) \ + _Py_atomic_load_ullong_relaxed(&value) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -68,7 +116,6 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE(value) value #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value -#define FT_ATOMIC_STORE_PTR(value, new_value) value = new_value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value @@ -85,6 +132,30 @@ extern "C" { #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value +#define FT_ATOMIC_STORE_UCHAR_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_SHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value +#define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT_RELAXED(value) value +#define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT_RELAXED(value) value +#define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_ULONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) value +#define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_DOUBLE_RELAXED(value) value +#define FT_ATOMIC_STORE_DOUBLE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_LLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_LLONG_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value +#define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #endif diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 54d8803bc0bdb6..1e73e541ef8de0 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -190,10 +190,18 @@ static inline void _Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) { if (tstate == NULL) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "the function must be called with the GIL held, " "after Python initialization and before Python finalization, " "but the GIL is released (the current Python thread state is NULL)"); +#else + _Py_FatalErrorFunc(func, + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 9f6748945bab36..1260b957ce9482 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -137,6 +137,7 @@ extern PyTypeObject _PyExc_MemoryError; { .threshold = 0, }, \ }, \ .work_to_do = -5000, \ + .phase = GC_PHASE_MARK, \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 1c1f478c3833c8..89fce193f40bd8 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -84,7 +84,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_SUBSCR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG, + [_BINARY_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, [_BINARY_SUBSCR_STR_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_TUPLE_INT] = HAS_DEOPT_FLAG, [_BINARY_SUBSCR_DICT] = HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -112,7 +112,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_UNPACK_SEQUENCE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_UNPACK_SEQUENCE_TWO_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_SEQUENCE_TUPLE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, + [_UNPACK_SEQUENCE_LIST] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_UNPACK_EX] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_DELETE_ATTR] = HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Include/object.h b/Include/object.h index 3876d8449afbe2..da7b3668c033f4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -71,7 +71,7 @@ whose size is determined when the object is allocated. #define PyObject_HEAD_INIT(type) \ { \ 0, \ - 0, \ + _Py_STATICALLY_ALLOCATED_FLAG, \ { 0 }, \ 0, \ _Py_IMMORTAL_REFCNT_LOCAL, \ @@ -81,7 +81,7 @@ whose size is determined when the object is allocated. #else #define PyObject_HEAD_INIT(type) \ { \ - { _Py_IMMORTAL_INITIAL_REFCNT }, \ + { _Py_STATIC_IMMORTAL_INITIAL_REFCNT }, \ (type) \ }, #endif @@ -120,9 +120,19 @@ struct _object { __pragma(warning(disable: 4201)) #endif union { - Py_ssize_t ob_refcnt; #if SIZEOF_VOID_P > 4 - PY_UINT32_T ob_refcnt_split[2]; + PY_INT64_T ob_refcnt_full; /* This field is needed for efficient initialization with Clang on ARM */ + struct { +# if PY_BIG_ENDIAN + PY_UINT32_T ob_flags; + PY_UINT32_T ob_refcnt; +# else + PY_UINT32_T ob_refcnt; + PY_UINT32_T ob_flags; +# endif + }; +#else + Py_ssize_t ob_refcnt; #endif }; #ifdef _MSC_VER @@ -142,7 +152,7 @@ struct _object { // trashcan mechanism as a linked list pointer and by the GC to store the // computed "gc_refs" refcount. uintptr_t ob_tid; - uint16_t _padding; + uint16_t ob_flags; PyMutex ob_mutex; // per-object lock uint8_t ob_gc_bits; // gc-related state uint32_t ob_ref_local; // local reference count diff --git a/Include/refcount.h b/Include/refcount.h index 141cbd34dd72e6..6908c426141378 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -19,6 +19,9 @@ immortal. The latter should be the only instances that require cleanup during runtime finalization. */ +/* Leave the low bits for refcount overflow for old stable ABI code */ +#define _Py_STATICALLY_ALLOCATED_FLAG (1 << 7) + #if SIZEOF_VOID_P > 4 /* In 64+ bit systems, any object whose 32 bit reference count is >= 2**31 @@ -39,7 +42,8 @@ beyond the refcount limit. Immortality checks for reference count decreases will be done by checking the bit sign flag in the lower 32 bits. */ -#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3UL << 30)) +#define _Py_IMMORTAL_INITIAL_REFCNT (3UL << 30) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(_Py_IMMORTAL_INITIAL_REFCNT | (((Py_ssize_t)_Py_STATICALLY_ALLOCATED_FLAG) << 32))) #else /* @@ -54,8 +58,10 @@ immortality, but the execution would still be correct. Reference count increases and decreases will first go through an immortality check by comparing the reference count field to the minimum immortality refcount. */ -#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(3L << 29)) +#define _Py_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(5L << 28)) #define _Py_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(1L << 30)) +#define _Py_STATIC_IMMORTAL_INITIAL_REFCNT ((Py_ssize_t)(7L << 28)) +#define _Py_STATIC_IMMORTAL_MINIMUM_REFCNT ((Py_ssize_t)(6L << 28)) #endif // Py_GIL_DISABLED builds indicate immortal objects using `ob_ref_local`, which is @@ -123,10 +129,21 @@ static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) #define _Py_IsImmortal(op) _Py_IsImmortal(_PyObject_CAST(op)) +static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) +{ +#if defined(Py_GIL_DISABLED) || SIZEOF_VOID_P > 4 + return (op->ob_flags & _Py_STATICALLY_ALLOCATED_FLAG) != 0; +#else + return op->ob_refcnt >= _Py_STATIC_IMMORTAL_MINIMUM_REFCNT; +#endif +} +#define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) + // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { + assert(refcnt >= 0); #if defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030d0000 // Stable ABI implements Py_SET_REFCNT() as a function call // on limited C API version 3.13 and newer. @@ -139,9 +156,12 @@ static inline void Py_SET_REFCNT(PyObject *ob, Py_ssize_t refcnt) { if (_Py_IsImmortal(ob)) { return; } - #ifndef Py_GIL_DISABLED +#if SIZEOF_VOID_P > 4 + ob->ob_refcnt = (PY_UINT32_T)refcnt; +#else ob->ob_refcnt = refcnt; +#endif #else if (_Py_IsOwnedByCurrentThread(ob)) { if ((size_t)refcnt > (size_t)UINT32_MAX) { @@ -252,13 +272,13 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) _Py_atomic_add_ssize(&op->ob_ref_shared, (1 << _Py_REF_SHARED_SHIFT)); } #elif SIZEOF_VOID_P > 4 - PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN]; + PY_UINT32_T cur_refcnt = op->ob_refcnt; if (((int32_t)cur_refcnt) < 0) { // the object is immortal _Py_INCREF_IMMORTAL_STAT_INC(); return; } - op->ob_refcnt_split[PY_BIG_ENDIAN] = cur_refcnt + 1; + op->ob_refcnt = cur_refcnt + 1; #else if (_Py_IsImmortal(op)) { _Py_INCREF_IMMORTAL_STAT_INC(); @@ -354,7 +374,13 @@ static inline void Py_DECREF(PyObject *op) #elif defined(Py_REF_DEBUG) static inline void Py_DECREF(const char *filename, int lineno, PyObject *op) { +#if SIZEOF_VOID_P > 4 + /* If an object has been freed, it will have a negative full refcnt + * If it has not it been freed, will have a very large refcnt */ + if (op->ob_refcnt_full <= 0 || op->ob_refcnt > (UINT32_MAX - (1<<20))) { +#else if (op->ob_refcnt <= 0) { +#endif _Py_NegativeRefcount(filename, lineno, op); } if (_Py_IsImmortal(op)) { diff --git a/InternalDocs/README.md b/InternalDocs/README.md index f6aa3db3b384af..794b4f3c6aad42 100644 --- a/InternalDocs/README.md +++ b/InternalDocs/README.md @@ -1,4 +1,3 @@ - # CPython Internals Documentation The documentation in this folder is intended for CPython maintainers. @@ -35,9 +34,9 @@ Runtime Objects Program Execution --- -- [The Interpreter](interpreter.md) +- [The Bytecode Interpreter](interpreter.md) -- [Adaptive Instruction Families](adaptive.md) +- [The JIT](jit.md) - [Garbage Collector Design](garbage_collector.md) diff --git a/InternalDocs/adaptive.md b/InternalDocs/adaptive.md deleted file mode 100644 index 4ae9e85b387f39..00000000000000 --- a/InternalDocs/adaptive.md +++ /dev/null @@ -1,142 +0,0 @@ -# Adding or extending a family of adaptive instructions. - -## Families of instructions - -The core part of [PEP 659](https://peps.python.org/pep-0659/) -(specializing adaptive interpreter) is the families of -instructions that perform the adaptive specialization. - -A family of instructions has the following fundamental properties: - -* It corresponds to a single instruction in the code - generated by the bytecode compiler. -* It has a single adaptive instruction that records an execution count and, - at regular intervals, attempts to specialize itself. If not specializing, - it executes the base implementation. -* It has at least one specialized form of the instruction that is tailored - for a particular value or set of values at runtime. -* All members of the family must have the same number of inline cache entries, - to ensure correct execution. - Individual family members do not need to use all of the entries, - but must skip over any unused entries when executing. - -The current implementation also requires the following, -although these are not fundamental and may change: - -* All families use one or more inline cache entries, - the first entry is always the counter. -* All instruction names should start with the name of the adaptive - instruction. -* Specialized forms should have names describing their specialization. - -## Example family - -The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) -already has an adaptive family that serves as a relatively simple example. - -The `LOAD_GLOBAL` instruction performs adaptive specialization, -calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. - -There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` -which is specialized for global variables in the module, and -`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. - -## Performance analysis - -The benefit of a specialization can be assessed with the following formula: -`Tbase/Tadaptive`. - -Where `Tbase` is the mean time to execute the base instruction, -and `Tadaptive` is the mean time to execute the specialized and adaptive forms. - -`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` - -`Ti` is the time to execute the `i`th instruction in the family and `Ni` is -the number of times that instruction is executed. -`Tmiss` is the time to process a miss, including de-optimzation -and the time to execute the base instruction. - -The ideal situation is where misses are rare and the specialized -forms are much faster than the base instruction. -`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. -In which case we have `Tadaptive ≈ sum(Ti*Ni)`. -Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and -`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, -we would expect the specialization of `LOAD_GLOBAL` to be profitable. - -## Design considerations - -While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and -`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` -low for all specialized instructions and `Nmiss` as low as possible. - -Keeping `Nmiss` low means that there should be specializations for almost -all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means -keeping `Ti` low which means minimizing branches and dependent memory -accesses (pointer chasing). These two objectives may be in conflict, -requiring judgement and experimentation to design the family of instructions. - -The size of the inline cache should as small as possible, -without impairing performance, to reduce the number of -`EXTENDED_ARG` jumps, and to reduce pressure on the CPU's data cache. - -### Gathering data - -Before choosing how to specialize an instruction, it is important to gather -some data. What are the patterns of usage of the base instruction? -Data can best be gathered by instrumenting the interpreter. Since a -specialization function and adaptive instruction are going to be required, -instrumentation can most easily be added in the specialization function. - -### Choice of specializations - -The performance of the specializing adaptive interpreter relies on the -quality of specialization and keeping the overhead of specialization low. - -Specialized instructions must be fast. In order to be fast, -specialized instructions should be tailored for a particular -set of values that allows them to: -1. Verify that incoming value is part of that set with low overhead. -2. Perform the operation quickly. - -This requires that the set of values is chosen such that membership can be -tested quickly and that membership is sufficient to allow the operation to -performed quickly. - -For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` -dictionaries that have a keys with the expected version. - -This can be tested quickly: -* `globals->keys->dk_version == expected_version` - -and the operation can be performed quickly: -* `value = entries[cache->index].me_value;`. - -Because it is impossible to measure the performance of an instruction without -also measuring unrelated factors, the assessment of the quality of a -specialization will require some judgement. - -As a general rule, specialized instructions should be much faster than the -base instruction. - -### Implementation of specialized instructions - -In general, specialized instructions should be implemented in two parts: -1. A sequence of guards, each of the form - `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. -2. The operation, which should ideally have no branches and - a minimum number of dependent memory accesses. - -In practice, the parts may overlap, as data required for guards -can be re-used in the operation. - -If there are branches in the operation, then consider further specialization -to eliminate the branches. - -### Maintaining stats - -Finally, take care that stats are gather correctly. -After the last `DEOPT_IF` has passed, a hit should be recorded with -`STAT_INC(BASE_INSTRUCTION, hit)`. -After an optimization has been deferred in the adaptive instruction, -that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. diff --git a/InternalDocs/changing_grammar.md b/InternalDocs/changing_grammar.md index 1a5eebdc1418dc..c6b895135a360d 100644 --- a/InternalDocs/changing_grammar.md +++ b/InternalDocs/changing_grammar.md @@ -32,7 +32,7 @@ Below is a checklist of things that may need to change. [`Include/internal/pycore_ast.h`](../Include/internal/pycore_ast.h) and [`Python/Python-ast.c`](../Python/Python-ast.c). -* [`Parser/lexer/`](../Parser/lexer/) contains the tokenization code. +* [`Parser/lexer/`](../Parser/lexer) contains the tokenization code. This is where you would add a new type of comment or string literal, for example. * [`Python/ast.c`](../Python/ast.c) will need changes to validate AST objects @@ -60,4 +60,4 @@ Below is a checklist of things that may need to change. to the tokenizer. * Documentation must be written! Specifically, one or more of the pages in - [`Doc/reference/`](../Doc/reference/) will need to be updated. + [`Doc/reference/`](../Doc/reference) will need to be updated. diff --git a/InternalDocs/code_objects.md b/InternalDocs/code_objects.md index bee4a9d0a08915..a91a7043c1b8d4 100644 --- a/InternalDocs/code_objects.md +++ b/InternalDocs/code_objects.md @@ -1,4 +1,3 @@ - # Code objects A `CodeObject` is a builtin Python type that represents a compiled executable, @@ -19,6 +18,11 @@ Code objects are typically produced by the bytecode [compiler](compiler.md), although they are often written to disk by one process and read back in by another. The disk version of a code object is serialized using the [marshal](https://docs.python.org/dev/library/marshal.html) protocol. +When a `CodeObject` is created, the function `_PyCode_Quicken()` from +[`Python/specialize.c`](../Python/specialize.c) is called to initialize +the caches of all adaptive instructions. This is required because the +on-disk format is a sequence of bytes, and some of the caches need to be +initialized with 16-bit values. Code objects are nominally immutable. Some fields (including `co_code_adaptive` and fields for runtime @@ -43,7 +47,7 @@ so a compact format is very important. Note that traceback objects don't store all this information -- they store the start line number, for backward compatibility, and the "last instruction" value. The rest can be computed from the last instruction (`tb_lasti`) with the help of the -locations table. For Python code, there is a convenience method +locations table. For Python code, there is a convenience method (`codeobject.co_positions`)[https://docs.python.org/dev/reference/datamodel.html#codeobject.co_positions] which returns an iterator of `({line}, {endline}, {column}, {endcolumn})` tuples, one per instruction. @@ -75,9 +79,11 @@ returned by the `co_positions()` iterator. > See [`Objects/lnotab_notes.txt`](../Objects/lnotab_notes.txt) for more details. `co_linetable` consists of a sequence of location entries. -Each entry starts with a byte with the most significant bit set, followed by zero or more bytes with the most significant bit unset. +Each entry starts with a byte with the most significant bit set, followed by +zero or more bytes with the most significant bit unset. Each entry contains the following information: + * The number of code units covered by this entry (length) * The start line * The end line @@ -86,54 +92,88 @@ Each entry contains the following information: The first byte has the following format: -Bit 7 | Bits 3-6 | Bits 0-2 - ---- | ---- | ---- - 1 | Code | Length (in code units) - 1 +| Bit 7 | Bits 3-6 | Bits 0-2 | +|-------|----------|----------------------------| +| 1 | Code | Length (in code units) - 1 | The codes are enumerated in the `_PyCodeLocationInfoKind` enum. -## Variable-length integer encodings +### Variable-length integer encodings -Integers are often encoded using a variable-length integer encoding +Integers are often encoded using a variable length integer encoding -### Unsigned integers (`varint`) +#### Unsigned integers (`varint`) Unsigned integers are encoded in 6-bit chunks, least significant first. Each chunk but the last has bit 6 set. For example: * 63 is encoded as `0x3f` -* 200 is encoded as `0x48`, `0x03` +* 200 is encoded as `0x48`, `0x03` since ``200 = (0x03 << 6) | 0x48``. + +The following helper can be used to convert an integer into a `varint`: + +```py +def encode_varint(s): + ret = [] + while s >= 64: + ret.append(((s & 0x3F) | 0x40) & 0x3F) + s >>= 6 + ret.append(s & 0x3F) + return bytes(ret) +``` + +To convert a `varint` into an unsigned integer: + +```py +def decode_varint(chunks): + ret = 0 + for chunk in reversed(chunks): + ret = (ret << 6) | chunk + return ret +``` -### Signed integers (`svarint`) +#### Signed integers (`svarint`) Signed integers are encoded by converting them to unsigned integers, using the following function: -```Python -def convert(s): + +```py +def svarint_to_varint(s): if s < 0: - return ((-s)<<1) | 1 + return ((-s) << 1) | 1 else: - return (s<<1) + return s << 1 +``` + +To convert a `varint` into a signed integer: + +```py +def varint_to_svarint(uval): + return -(uval >> 1) if uval & 1 else (uval >> 1) ``` -*Location entries* +### Location entries The meaning of the codes and the following bytes are as follows: -Code | Meaning | Start line | End line | Start column | End column - ---- | ---- | ---- | ---- | ---- | ---- - 0-9 | Short form | Δ 0 | Δ 0 | See below | See below - 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte - 13 | No column info | Δ svarint | Δ 0 | None | None - 14 | Long form | Δ svarint | Δ varint | varint | varint - 15 | No location | None | None | None | None +| Code | Meaning | Start line | End line | Start column | End column | +|-------|----------------|---------------|----------|---------------|---------------| +| 0-9 | Short form | Δ 0 | Δ 0 | See below | See below | +| 10-12 | One line form | Δ (code - 10) | Δ 0 | unsigned byte | unsigned byte | +| 13 | No column info | Δ svarint | Δ 0 | None | None | +| 14 | Long form | Δ svarint | Δ varint | varint | varint | +| 15 | No location | None | None | None | None | The Δ means the value is encoded as a delta from another value: + * Start line: Delta from the previous start line, or `co_firstlineno` for the first entry. -* End line: Delta from the start line +* End line: Delta from the start line. + +### The short forms -*The short forms* +Codes 0-9 are the short forms. The short form consists of two bytes, +the second byte holding additional column information. The code is the +start column divided by 8 (and rounded down). -Codes 0-9 are the short forms. The short form consists of two bytes, the second byte holding additional column information. The code is the start column divided by 8 (and rounded down). * Start column: `(code*8) + ((second_byte>>4)&7)` * End column: `start_column + (second_byte&15)` diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index ed4cfb23ca51f7..c257bfd9faf78f 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -1,4 +1,3 @@ - Compiler design =============== @@ -7,8 +6,8 @@ Abstract In CPython, the compilation from source code to bytecode involves several steps: -1. Tokenize the source code [Parser/lexer/](../Parser/lexer/) - and [Parser/tokenizer/](../Parser/tokenizer/). +1. Tokenize the source code [Parser/lexer/](../Parser/lexer) + and [Parser/tokenizer/](../Parser/tokenizer). 2. Parse the stream of tokens into an Abstract Syntax Tree [Parser/parser.c](../Parser/parser.c). 3. Transform AST into an instruction sequence @@ -134,9 +133,8 @@ this case) a `stmt_ty` struct with the appropriate initialization. The `FunctionDef()` constructor function sets 'kind' to `FunctionDef_kind` and initializes the *name*, *args*, *body*, and *attributes* fields. -See also -[Green Tree Snakes - The missing Python AST docs](https://greentreesnakes.readthedocs.io/en/latest) - by Thomas Kluyver. +See also [Green Tree Snakes - The missing Python AST docs]( +https://greentreesnakes.readthedocs.io/en/latest) by Thomas Kluyver. Memory management ================= @@ -260,12 +258,12 @@ manually -- `generic`, `identifier` and `int`. These types are found in [Include/internal/pycore_asdl.h](../Include/internal/pycore_asdl.h). Functions and macros for creating `asdl_xx_seq *` types are as follows: -`_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_generic_seq` of the specified length -`_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_identifier_seq` of the specified length -`_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)` - Allocate memory for an `asdl_int_seq` of the specified length +* `_Py_asdl_generic_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_generic_seq` of the specified length +* `_Py_asdl_identifier_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_identifier_seq` of the specified length +* `_Py_asdl_int_seq_new(Py_ssize_t, PyArena *)`: + Allocate memory for an `asdl_int_seq` of the specified length In addition to the three types mentioned above, some ASDL sequence types are automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in @@ -273,20 +271,20 @@ automatically generated by [Parser/asdl_c.py](../Parser/asdl_c.py) and found in Macros for using both manually defined and automatically generated ASDL sequence types are as follows: -`asdl_seq_GET(asdl_xx_seq *, int)` - Get item held at a specific position in an `asdl_xx_seq` -`asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)` - Set a specific index in an `asdl_xx_seq` to the specified value +* `asdl_seq_GET(asdl_xx_seq *, int)`: + Get item held at a specific position in an `asdl_xx_seq` +* `asdl_seq_SET(asdl_xx_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_xx_seq` to the specified value -Untyped counterparts exist for some of the typed macros. These are useful +Untyped counterparts exist for some of the typed macros. These are useful when a function needs to manipulate a generic ASDL sequence: -`asdl_seq_GET_UNTYPED(asdl_seq *, int)` - Get item held at a specific position in an `asdl_seq` -`asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)` - Set a specific index in an `asdl_seq` to the specified value -`asdl_seq_LEN(asdl_seq *)` - Return the length of an `asdl_seq` or `asdl_xx_seq` +* `asdl_seq_GET_UNTYPED(asdl_seq *, int)`: + Get item held at a specific position in an `asdl_seq` +* `asdl_seq_SET_UNTYPED(asdl_seq *, int, stmt_ty)`: + Set a specific index in an `asdl_seq` to the specified value +* `asdl_seq_LEN(asdl_seq *)`: + Return the length of an `asdl_seq` or `asdl_xx_seq` Note that typed macros and functions are recommended over their untyped counterparts. Typed macros carry out checks in debug mode and aid @@ -379,33 +377,33 @@ arguments to a node that used the '*' modifier). Emission of bytecode is handled by the following macros: -* `ADDOP(struct compiler *, location, int)` - add a specified opcode -* `ADDOP_IN_SCOPE(struct compiler *, location, int)` - like `ADDOP`, but also exits current scope; used for adding return value - opcodes in lambdas and closures -* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)` - add an opcode that takes an integer argument -* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)` - add an opcode with the proper argument based on the position of the - specified PyObject in PyObject sequence object, but with no handling of - mangled names; used for when you - need to do named lookups of objects such as globals, consts, or - parameters where name mangling is not possible and the scope of the - name is known; *TYPE* is the name of PyObject sequence - (`names` or `varnames`) -* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)` - just like `ADDOP_O`, but steals a reference to PyObject -* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)` - just like `ADDOP_O`, but name mangling is also handled; used for - attribute loading or importing based on name -* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)` - add the `LOAD_CONST` opcode with the proper argument based on the - position of the specified PyObject in the consts table. -* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)` - just like `ADDOP_LOAD_CONST_NEW`, but steals a reference to PyObject -* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)` - create a jump to a basic block +* `ADDOP(struct compiler *, location, int)`: + add a specified opcode +* `ADDOP_IN_SCOPE(struct compiler *, location, int)`: + like `ADDOP`, but also exits current scope; used for adding return value + opcodes in lambdas and closures +* `ADDOP_I(struct compiler *, location, int, Py_ssize_t)`: + add an opcode that takes an integer argument +* `ADDOP_O(struct compiler *, location, int, PyObject *, TYPE)`: + add an opcode with the proper argument based on the position of the + specified PyObject in PyObject sequence object, but with no handling of + mangled names; used for when you + need to do named lookups of objects such as globals, consts, or + parameters where name mangling is not possible and the scope of the + name is known; *TYPE* is the name of PyObject sequence + (`names` or `varnames`) +* `ADDOP_N(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but steals a reference to PyObject +* `ADDOP_NAME(struct compiler *, location, int, PyObject *, TYPE)`: + just like `ADDOP_O`, but name mangling is also handled; used for + attribute loading or importing based on name +* `ADDOP_LOAD_CONST(struct compiler *, location, PyObject *)`: + add the `LOAD_CONST` opcode with the proper argument based on the + position of the specified PyObject in the consts table. +* `ADDOP_LOAD_CONST_NEW(struct compiler *, location, PyObject *)`: + just like `ADDOP_LOAD_CONST_NEW`, but steals a reference to PyObject +* `ADDOP_JUMP(struct compiler *, location, int, basicblock *)`: + create a jump to a basic block The `location` argument is a struct with the source location to be associated with this instruction. It is typically extracted from an @@ -433,7 +431,7 @@ Finally, the sequence of pseudo-instructions is converted into actual bytecode. This includes transforming pseudo instructions into actual instructions, converting jump targets from logical labels to relative offsets, and construction of the [exception table](exception_handling.md) and -[locations table](locations.md). +[locations table](code_objects.md#source-code-locations). The bytecode and tables are then wrapped into a `PyCodeObject` along with additional metadata, including the `consts` and `names` arrays, information about function reference to the source code (filename, etc). All of this is implemented by @@ -453,7 +451,7 @@ in [Python/ceval.c](../Python/ceval.c). Important files =============== -* [Parser/](../Parser/) +* [Parser/](../Parser) * [Parser/Python.asdl](../Parser/Python.asdl): ASDL syntax file. @@ -534,7 +532,7 @@ Important files * [Python/instruction_sequence.c](../Python/instruction_sequence.c): A data structure representing a sequence of bytecode-like pseudo-instructions. -* [Include/](../Include/) +* [Include/](../Include) * [Include/cpython/code.h](../Include/cpython/code.h) : Header file for [Objects/codeobject.c](../Objects/codeobject.c); @@ -556,7 +554,7 @@ Important files : Declares `_PyAST_Validate()` external (from [Python/ast.c](../Python/ast.c)). * [Include/internal/pycore_symtable.h](../Include/internal/pycore_symtable.h) - : Header for [Python/symtable.c](../Python/symtable.c). + : Header for [Python/symtable.c](../Python/symtable.c). `struct symtable` and `PySTEntryObject` are defined here. * [Include/internal/pycore_parser.h](../Include/internal/pycore_parser.h) @@ -570,7 +568,7 @@ Important files by [Tools/cases_generator/opcode_id_generator.py](../Tools/cases_generator/opcode_id_generator.py). -* [Objects/](../Objects/) +* [Objects/](../Objects) * [Objects/codeobject.c](../Objects/codeobject.c) : Contains PyCodeObject-related code. @@ -579,7 +577,7 @@ Important files : Contains the `frame_setlineno()` function which should determine whether it is allowed to make a jump between two points in a bytecode. -* [Lib/](../Lib/) +* [Lib/](../Lib) * [Lib/opcode.py](../Lib/opcode.py) : opcode utilities exposed to Python. @@ -591,22 +589,12 @@ Important files Objects ======= -* [Locations](locations.md): Describes the location table +* [Locations](code_objects.md#source-code-locations): Describes the location table * [Frames](frames.md): Describes frames and the frame stack * [Objects/object_layout.md](../Objects/object_layout.md): Describes object layout for 3.11 and later * [Exception Handling](exception_handling.md): Describes the exception table -Specializing Adaptive Interpreter -================================= - -Adding a specializing, adaptive interpreter to CPython will bring significant -performance improvements. These documents provide more information: - -* [PEP 659: Specializing Adaptive Interpreter](https://peps.python.org/pep-0659/). -* [Adding or extending a family of adaptive instructions](adaptive.md) - - References ========== diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 14066a5864b4da..28589787e1fad7 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -87,10 +87,10 @@ offset of the raising instruction should be pushed to the stack. Handling an exception, once an exception table entry is found, consists of the following steps: - 1. pop values from the stack until it matches the stack depth for the handler. - 2. if `lasti` is true, then push the offset that the exception was raised at. - 3. push the exception to the stack. - 4. jump to the target offset and resume execution. +1. pop values from the stack until it matches the stack depth for the handler. +2. if `lasti` is true, then push the offset that the exception was raised at. +3. push the exception to the stack. +4. jump to the target offset and resume execution. Reraising Exceptions and `lasti` @@ -107,13 +107,12 @@ Format of the exception table ----------------------------- Conceptually, the exception table consists of a sequence of 5-tuples: -``` - 1. `start-offset` (inclusive) - 2. `end-offset` (exclusive) - 3. `target` - 4. `stack-depth` - 5. `push-lasti` (boolean) -``` + +1. `start-offset` (inclusive) +2. `end-offset` (exclusive) +3. `target` +4. `stack-depth` +5. `push-lasti` (boolean) All offsets and lengths are in code units, not bytes. @@ -123,18 +122,19 @@ For it to be searchable quickly, we need to support binary search giving us log( Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: - `start, size, target, depth, push-lasti`. +`start, size, target, depth, push-lasti`. Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. It also happens that depth is generally quite small. So, we need to encode: + ``` - `start` (up to 30 bits) - `size` (up to 30 bits) - `target` (up to 30 bits) - `depth` (up to ~8 bits) - `lasti` (1 bit) +start (up to 30 bits) +size (up to 30 bits) +target (up to 30 bits) +depth (up to ~8 bits) +lasti (1 bit) ``` We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. @@ -145,29 +145,32 @@ The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. For example, the exception entry: + ``` - `start`: 20 - `end`: 28 - `target`: 100 - `depth`: 3 - `lasti`: False +start: 20 +end: 28 +target: 100 +depth: 3 +lasti: False ``` is encoded by first converting to the more compact four value form: + ``` - `start`: 20 - `size`: 8 - `target`: 100 - `depth<<1+lasti`: 6 +start: 20 +size: 8 +target: 100 +depth<<1+lasti: 6 ``` which is then encoded as: + ``` - 148 (MSB + 20 for start) - 8 (size) - 65 (Extend bit + 1) - 36 (Remainder of target, 100 == (1<<6)+36) - 6 +148 (MSB + 20 for start) +8 (size) +65 (Extend bit + 1) +36 (Remainder of target, 100 == (1<<6)+36) +6 ``` for a total of five bytes. diff --git a/InternalDocs/frames.md b/InternalDocs/frames.md index 06dc8f0702c3d9..2598873ca98479 100644 --- a/InternalDocs/frames.md +++ b/InternalDocs/frames.md @@ -27,6 +27,7 @@ objects, so are not allocated in the per-thread stack. See `PyGenObject` in ## Layout Each activation record is laid out as: + * Specials * Locals * Stack diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 272a0834cbfe24..e4cb9e45c9e96a 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -1,4 +1,3 @@ - Garbage collector design ======================== @@ -117,7 +116,7 @@ general, the collection of all objects tracked by GC is partitioned into disjoin doubly linked list. Between collections, objects are partitioned into "generations", reflecting how often they've survived collection attempts. During collections, the generation(s) being collected are further partitioned into, for example, sets of reachable and unreachable objects. Doubly linked lists -support moving an object from one partition to another, adding a new object, removing an object +support moving an object from one partition to another, adding a new object, removing an object entirely (objects tracked by GC are most often reclaimed by the refcounting system when GC isn't running at all!), and merging partitions, all with a small constant number of pointer updates. With care, they also support iterating over a partition while objects are being added to - and @@ -478,6 +477,44 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ``` +Optimization: visiting reachable objects +======================================== + +An object cannot be garbage if it can be reached. + +To avoid having to identify reference cycles across the whole heap, we can +reduce the amount of work done considerably by first moving most reachable objects +to the `visited` space. Empirically, most reachable objects can be reached from a +small set of global objects and local variables. +This step does much less work per object, so reduces the time spent +performing garbage collection by at least half. + +> [!NOTE] +> Objects that are not determined to be reachable by this pass are not necessarily +> unreachable. We still need to perform the main algorithm to determine which objects +> are actually unreachable. +We use the same technique of forming a transitive closure as the incremental +collector does to find reachable objects, seeding the list with some global +objects and the currently executing frames. + +This phase moves objects to the `visited` space, as follows: + +1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` +module and all objects directly referred to from stack frames are added to a working +set of reachable objects. +2. Until this working set is empty: + 1. Pop an object from the set and move it to the `visited` space + 2. For each object directly reachable from that object: + * If it is not already in `visited` space and it is a GC object, + add it to the working set + + +Before each increment of collection is performed, the stacks are scanned +to check for any new stack frames that have been created since the last +increment. All objects directly referred to from those stack frames are +added to the working set. +Then the above algorithm is repeated, starting from step 2. + Optimization: reusing fields to save memory =========================================== diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index d53f0f9bdff4e4..afa8b8f4bb8040 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -1,4 +1,3 @@ - Generators ========== diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 4c10cbbed37735..fa4a54fdc54fac 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -1,9 +1,4 @@ - -The bytecode interpreter -======================== - -Overview --------- +# The bytecode interpreter This document describes the workings and implementation of the bytecode interpreter, the part of python that executes compiled Python code. Its @@ -48,8 +43,7 @@ simply calls [`_PyEval_EvalFrameDefault()`] to execute the frame. However, as pe `_PyEval_EvalFrameDefault()`. -Instruction decoding --------------------- +## Instruction decoding The first task of the interpreter is to decode the bytecode instructions. Bytecode is stored as an array of 16-bit code units (`_Py_CODEUNIT`). @@ -111,8 +105,7 @@ snippet decode a complete instruction: For various reasons we'll get to later (mostly efficiency, given that `EXTENDED_ARG` is rare) the actual code is different. -Jumps -===== +## Jumps Note that when the `switch` statement is reached, `next_instr` (the "instruction offset") already points to the next instruction. @@ -121,25 +114,26 @@ Thus, jump instructions can be implemented by manipulating `next_instr`: - A jump forward (`JUMP_FORWARD`) sets `next_instr += oparg`. - A jump backward sets `next_instr -= oparg`. -Inline cache entries -==================== +## Inline cache entries Some (specialized or specializable) instructions have an associated "inline cache". The inline cache consists of one or more two-byte entries included in the bytecode array as additional words following the `opcode`/`oparg` pair. The size of the inline cache for a particular instruction is fixed by its `opcode`. Moreover, the inline cache size for all instructions in a -[family of specialized/specializable instructions](adaptive.md) +[family of specialized/specializable instructions](#Specialization) (for example, `LOAD_ATTR`, `LOAD_ATTR_SLOT`, `LOAD_ATTR_MODULE`) must all be the same. Cache entries are reserved by the compiler and initialized with zeros. Although they are represented by code units, cache entries do not conform to the `opcode` / `oparg` format. -If an instruction has an inline cache, the layout of its cache is described by -a `struct` definition in (`pycore_code.h`)[../Include/internal/pycore_code.h]. -This allows us to access the cache by casting `next_instr` to a pointer to this `struct`. -The size of such a `struct` must be independent of the machine architecture, word size -and alignment requirements. For a 32-bit field, the `struct` should use `_Py_CODEUNIT field[2]`. +If an instruction has an inline cache, the layout of its cache is described in +the instruction's definition in [`Python/bytecodes.c`](../Python/bytecodes.c). +The structs defined in [`pycore_code.h`](../Include/internal/pycore_code.h) +allow us to access the cache by casting `next_instr` to a pointer to the relevant +`struct`. The size of such a `struct` must be independent of the machine +architecture, word size and alignment requirements. For a 32-bit field, the +`struct` should use `_Py_CODEUNIT field[2]`. The instruction implementation is responsible for advancing `next_instr` past the inline cache. For example, if an instruction's inline cache is four bytes (that is, two code units) in size, @@ -154,8 +148,7 @@ Serializing non-zero cache entries would present a problem because the serializa More information about the use of inline caches can be found in [PEP 659](https://peps.python.org/pep-0659/#ancillary-data). -The evaluation stack --------------------- +## The evaluation stack Most instructions read or write some data in the form of object references (`PyObject *`). The CPython bytecode interpreter is a stack machine, meaning that its instructions operate @@ -194,16 +187,14 @@ For example, the following sequence is illegal, because it keeps pushing items o > Do not confuse the evaluation stack with the call stack, which is used to implement calling > and returning from functions. -Error handling --------------- +## Error handling When the implementation of an opcode raises an exception, it jumps to the `exception_unwind` label in [Python/ceval.c](../Python/ceval.c). The exception is then handled as described in the [`exception handling documentation`](exception_handling.md#handling-exceptions). -Python-to-Python calls ----------------------- +## Python-to-Python calls The `_PyEval_EvalFrameDefault()` function is recursive, because sometimes the interpreter calls some C function that calls back into the interpreter. @@ -228,8 +219,7 @@ returns from `_PyEval_EvalFrameDefault()` altogether, to a C caller. A similar check is performed when an unhandled exception occurs. -The call stack --------------- +## The call stack Up through 3.10, the call stack was implemented as a singly-linked list of [frame objects](frames.md). This was expensive because each call would require a @@ -263,8 +253,7 @@ See also the [generators](generators.md) section. -Introducing a new bytecode instruction --------------------------------------- +## Introducing a new bytecode instruction It is occasionally necessary to add a new opcode in order to implement a new feature or change the way that existing features are compiled. @@ -356,6 +344,169 @@ new bytecode properly. Run `make regen-importlib` for updating the bytecode of frozen importlib files. You have to run `make` again after this to recompile the generated C files. +## Specialization + +Bytecode specialization, which was introduced in +[PEP 659](https://peps.python.org/pep-0659/), speeds up program execution by +rewriting instructions based on runtime information. This is done by replacing +a generic instruction with a faster version that works for the case that this +program encounters. Each specializable instruction is responsible for rewriting +itself, using its [inline caches](#inline-cache-entries) for +bookkeeping. + +When an adaptive instruction executes, it may attempt to specialize itself, +depending on the argument and the contents of its cache. This is done +by calling one of the `_Py_Specialize_XXX` functions in +[`Python/specialize.c`](../Python/specialize.c). + + +The specialized instructions are responsible for checking that the special-case +assumptions still apply, and de-optimizing back to the generic version if not. + +## Families of instructions + +A *family* of instructions consists of an adaptive instruction along with the +specialized instructions that it can be replaced by. +It has the following fundamental properties: + +* It corresponds to a single instruction in the code + generated by the bytecode compiler. +* It has a single adaptive instruction that records an execution count and, + at regular intervals, attempts to specialize itself. If not specializing, + it executes the base implementation. +* It has at least one specialized form of the instruction that is tailored + for a particular value or set of values at runtime. +* All members of the family must have the same number of inline cache entries, + to ensure correct execution. + Individual family members do not need to use all of the entries, + but must skip over any unused entries when executing. + +The current implementation also requires the following, +although these are not fundamental and may change: + +* All families use one or more inline cache entries, + the first entry is always the counter. +* All instruction names should start with the name of the adaptive + instruction. +* Specialized forms should have names describing their specialization. + +## Example family + +The `LOAD_GLOBAL` instruction (in [Python/bytecodes.c](../Python/bytecodes.c)) +already has an adaptive family that serves as a relatively simple example. + +The `LOAD_GLOBAL` instruction performs adaptive specialization, +calling `_Py_Specialize_LoadGlobal()` when the counter reaches zero. + +There are two specialized instructions in the family, `LOAD_GLOBAL_MODULE` +which is specialized for global variables in the module, and +`LOAD_GLOBAL_BUILTIN` which is specialized for builtin variables. + +## Performance analysis + +The benefit of a specialization can be assessed with the following formula: +`Tbase/Tadaptive`. + +Where `Tbase` is the mean time to execute the base instruction, +and `Tadaptive` is the mean time to execute the specialized and adaptive forms. + +`Tadaptive = (sum(Ti*Ni) + Tmiss*Nmiss)/(sum(Ni)+Nmiss)` + +`Ti` is the time to execute the `i`th instruction in the family and `Ni` is +the number of times that instruction is executed. +`Tmiss` is the time to process a miss, including de-optimzation +and the time to execute the base instruction. + +The ideal situation is where misses are rare and the specialized +forms are much faster than the base instruction. +`LOAD_GLOBAL` is near ideal, `Nmiss/sum(Ni) ≈ 0`. +In which case we have `Tadaptive ≈ sum(Ti*Ni)`. +Since we can expect the specialized forms `LOAD_GLOBAL_MODULE` and +`LOAD_GLOBAL_BUILTIN` to be much faster than the adaptive base instruction, +we would expect the specialization of `LOAD_GLOBAL` to be profitable. + +## Design considerations + +While `LOAD_GLOBAL` may be ideal, instructions like `LOAD_ATTR` and +`CALL_FUNCTION` are not. For maximum performance we want to keep `Ti` +low for all specialized instructions and `Nmiss` as low as possible. + +Keeping `Nmiss` low means that there should be specializations for almost +all values seen by the base instruction. Keeping `sum(Ti*Ni)` low means +keeping `Ti` low which means minimizing branches and dependent memory +accesses (pointer chasing). These two objectives may be in conflict, +requiring judgement and experimentation to design the family of instructions. + +The size of the inline cache should as small as possible, +without impairing performance, to reduce the number of +`EXTENDED_ARG` jumps, and to reduce pressure on the CPU's data cache. + +### Gathering data + +Before choosing how to specialize an instruction, it is important to gather +some data. What are the patterns of usage of the base instruction? +Data can best be gathered by instrumenting the interpreter. Since a +specialization function and adaptive instruction are going to be required, +instrumentation can most easily be added in the specialization function. + +### Choice of specializations + +The performance of the specializing adaptive interpreter relies on the +quality of specialization and keeping the overhead of specialization low. + +Specialized instructions must be fast. In order to be fast, +specialized instructions should be tailored for a particular +set of values that allows them to: + +1. Verify that incoming value is part of that set with low overhead. +2. Perform the operation quickly. + +This requires that the set of values is chosen such that membership can be +tested quickly and that membership is sufficient to allow the operation to +performed quickly. + +For example, `LOAD_GLOBAL_MODULE` is specialized for `globals()` +dictionaries that have a keys with the expected version. + +This can be tested quickly: + +* `globals->keys->dk_version == expected_version` + +and the operation can be performed quickly: + +* `value = entries[cache->index].me_value;`. + +Because it is impossible to measure the performance of an instruction without +also measuring unrelated factors, the assessment of the quality of a +specialization will require some judgement. + +As a general rule, specialized instructions should be much faster than the +base instruction. + +### Implementation of specialized instructions + +In general, specialized instructions should be implemented in two parts: + +1. A sequence of guards, each of the form + `DEOPT_IF(guard-condition-is-false, BASE_NAME)`. +2. The operation, which should ideally have no branches and + a minimum number of dependent memory accesses. + +In practice, the parts may overlap, as data required for guards +can be re-used in the operation. + +If there are branches in the operation, then consider further specialization +to eliminate the branches. + +### Maintaining stats + +Finally, take care that stats are gathered correctly. +After the last `DEOPT_IF` has passed, a hit should be recorded with +`STAT_INC(BASE_INSTRUCTION, hit)`. +After an optimization has been deferred in the adaptive instruction, +that should be recorded with `STAT_INC(BASE_INSTRUCTION, deferred)`. + + Additional resources -------------------- diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md new file mode 100644 index 00000000000000..1e9f385d5f87fa --- /dev/null +++ b/InternalDocs/jit.md @@ -0,0 +1,134 @@ +# The JIT + +The [adaptive interpreter](interpreter.md) consists of a main loop that +executes the bytecode instructions generated by the +[bytecode compiler](compiler.md) and their +[specializations](interpreter.md#Specialization). Runtime optimization in +this interpreter can only be done for one instruction at a time. The JIT +is based on a mechanism to replace an entire sequence of bytecode instructions, +and this enables optimizations that span multiple instructions. + +Historically, the adaptive interpreter was referred to as `tier 1` and +the JIT as `tier 2`. You will see remnants of this in the code. + +## The Optimizer and Executors + +The program begins running on the adaptive interpreter, until a `JUMP_BACKWARD` +instruction determines that it is "hot" because the counter in its +[inline cache](interpreter.md#inline-cache-entries) indicates that it +executed more than some threshold number of times (see +[`backoff_counter_triggers`](../Include/internal/pycore_backoff.h)). +It then calls the function `_PyOptimizer_Optimize()` in +[`Python/optimizer.c`](../Python/optimizer.c), passing it the current +[frame](frames.md) and instruction pointer. `_PyOptimizer_Optimize()` +constructs an object of type +[`_PyExecutorObject`](Include/internal/pycore_optimizer.h) which implements +an optimized version of the instruction trace beginning at this jump. + +The optimizer determines where the trace ends, and the executor is set up +to either return to the adaptive interpreter and resume execution, or +transfer control to another executor (see `_PyExitData` in +Include/internal/pycore_optimizer.h). + +The executor is stored on the [`code object`](code_objects.md) of the frame, +in the `co_executors` field which is an array of executors. The start +instruction of the trace (the `JUMP_BACKWARD`) is replaced by an +`ENTER_EXECUTOR` instruction whose `oparg` is equal to the index of the +executor in `co_executors`. + +## The micro-op optimizer + +The optimizer that `_PyOptimizer_Optimize()` runs is configurable via the +`_Py_SetTier2Optimizer()` function (this is used in test via +`_testinternalcapi.set_optimizer()`.) + +The micro-op (abbreviated `uop` to approximate `μop`) optimizer is defined in +[`Python/optimizer.c`](../Python/optimizer.c) as the type `_PyUOpOptimizer_Type`. +It translates an instruction trace into a sequence of micro-ops by replacing +each bytecode by an equivalent sequence of micro-ops (see +`_PyOpcode_macro_expansion` in +[pycore_opcode_metadata.h](../Include/internal/pycore_opcode_metadata.h) +which is generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). +The micro-op sequence is then optimized by +`_Py_uop_analyze_and_optimize` in +[`Python/optimizer_analysis.c`](../Python/optimizer_analysis.c) +and an instance of `_PyUOpExecutor_Type` is created to contain it. + +## The JIT interpreter + +After a `JUMP_BACKWARD` instruction invokes the uop optimizer to create a uop +executor, it transfers control to this executor via the `GOTO_TIER_TWO` macro. + +CPython implements two executors. Here we describe the JIT interpreter, +which is the simpler of them and is therefore useful for debugging and analyzing +the uops generation and optimization stages. To run it, we configure the +JIT to run on its interpreter (i.e., python is configured with +[`--enable-experimental-jit=interpreter`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit)). + +When invoked, the executor jumps to the `tier2_dispatch:` label in +[`Python/ceval.c`](../Python/ceval.c), where there is a loop that +executes the micro-ops. The body of this loop is a switch statement over +the uops IDs, resembling the one used in the adaptive interpreter. + +The swtich implementing the uops is in [`Python/executor_cases.c.h`](../Python/executor_cases.c.h), +which is generated by the build script +[`Tools/cases_generator/tier2_generator.py`](../Tools/cases_generator/tier2_generator.py) +from the bytecode definitions in +[`Python/bytecodes.c`](../Python/bytecodes.c). + +When an `_EXIT_TRACE` or `_DEOPT` uop is reached, the uop interpreter exits +and execution returns to the adaptive interpreter. + +## Invalidating Executors + +In addition to being stored on the code object, each executor is also +inserted into a list of all executors, which is stored in the interpreter +state's `executor_list_head` field. This list is used when it is necessary +to invalidate executors because values they used in their construction may +have changed. + +## The JIT + +When the full jit is enabled (python was configured with +[`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), +the uop executor's `jit_code` field is populated with a pointer to a compiled +C function that implements the executor logic. This function's signature is +defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to +the uop interpreter at `tier2_dispatch`, the executor runs the function +that `jit_code` points to. This function returns the instruction pointer +of the next Tier 1 instruction that needs to execute. + +The generation of the jitted functions uses the copy-and-patch technique +which is described in +[Haoran Xu's article](https://sillycross.github.io/2023/05/12/2023-05-12/). +At its core are statically generated `stencils` for the implementation +of the micro ops, which are completed with runtime information while +the jitted code is constructed for an executor by +[`_PyJIT_Compile`](../Python/jit.c). + +The stencils are generated at build time under the Makefile target `regen-jit` +by the scripts in [`/Tools/jit`](/Tools/jit). This script reads +[`Python/executor_cases.c.h`](../Python/executor_cases.c.h) (which is +generated from [`Python/bytecodes.c`](../Python/bytecodes.c)). For +each opcode, it constructs a `.c` file that contains a function for +implementing this opcode, with some runtime information injected. +This is done by replacing `CASE` by the bytecode definition in the +template file [`Tools/jit/template.c`](../Tools/jit/template.c). + +Each of the `.c` files is compiled by LLVM, to produce an object file +that contains a function that executes the opcode. These compiled +functions are used to generate the file +[`jit_stencils.h`](../jit_stencils.h), which contains the functions +that the JIT can use to emit code for each of the bytecodes. + +For Python maintainers this means that changes to the bytecodes and +their implementations do not require changes related to the stencils, +because everything is automatically generated from +[`Python/bytecodes.c`](../Python/bytecodes.c) at build time. + +See Also: + +* [Copy-and-Patch Compilation: A fast compilation algorithm for high-level languages and bytecode](https://arxiv.org/abs/2011.13127) + +* [PyCon 2024: Building a JIT compiler for CPython](https://www.youtube.com/watch?v=kMO3Ju0QCDo) diff --git a/InternalDocs/parser.md b/InternalDocs/parser.md index 348988b7c2f003..445b866fc0cb96 100644 --- a/InternalDocs/parser.md +++ b/InternalDocs/parser.md @@ -1,4 +1,3 @@ - Guide to the parser =================== @@ -444,15 +443,15 @@ How to regenerate the parser Once you have made the changes to the grammar files, to regenerate the `C` parser (the one used by the interpreter) just execute: -``` - make regen-pegen +```shell +$ make regen-pegen ``` using the `Makefile` in the main directory. If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` The generated parser file is located at [`Parser/parser.c`](../Parser/parser.c). @@ -468,15 +467,15 @@ any modifications to this file (in order to implement new Pegen features) you wi need to regenerate the meta-parser (the parser that parses the grammar files). To do so just execute: -``` - make regen-pegen-metaparser +```shell +$ make regen-pegen-metaparser ``` If you are on Windows you can use the Visual Studio project files to regenerate the parser or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` @@ -516,15 +515,15 @@ be found in the [`Grammar/Tokens`](../Grammar/Tokens) file. If you change this file to add new tokens, make sure to regenerate the files by executing: -``` - make regen-token +```shell +$ make regen-token ``` If you are on Windows you can use the Visual Studio project files to regenerate the tokens or to execute: -``` - ./PCbuild/build.bat --regen +```dos +PCbuild/build.bat --regen ``` How tokens are generated and the rules governing this are completely up to the tokenizer @@ -546,8 +545,8 @@ by default** except for rules with the special marker `memo` after the rule name (and type, if present): ``` - rule_name[typr] (memo): - ... +rule_name[typr] (memo): + ... ``` By selectively turning on memoization for a handful of rules, the parser becomes @@ -593,25 +592,25 @@ are always reserved words, even in positions where they make no sense meaning in context. Trying to use a hard keyword as a variable will always fail: -``` - >>> class = 3 - File "", line 1 - class = 3 - ^ - SyntaxError: invalid syntax - >>> foo(class=3) - File "", line 1 - foo(class=3) - ^^^^^ - SyntaxError: invalid syntax +```pycon +>>> class = 3 +File "", line 1 + class = 3 + ^ +SyntaxError: invalid syntax +>>> foo(class=3) +File "", line 1 + foo(class=3) + ^^^^^ +SyntaxError: invalid syntax ``` While soft keywords don't have this limitation if used in a context other the one where they are defined as keywords: -``` - >>> match = 45 - >>> foo(match="Yeah!") +```pycon +>>> match = 45 +>>> foo(match="Yeah!") ``` The `match` and `case` keywords are soft keywords, so that they are @@ -621,21 +620,21 @@ argument names. You can get a list of all keywords defined in the grammar from Python: -``` - >>> import keyword - >>> keyword.kwlist - ['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', - 'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', - 'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', - 'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] +```pycon +>>> import keyword +>>> keyword.kwlist +['False', 'None', 'True', 'and', 'as', 'assert', 'async', 'await', 'break', +'class', 'continue', 'def', 'del', 'elif', 'else', 'except', 'finally', 'for', +'from', 'global', 'if', 'import', 'in', 'is', 'lambda', 'nonlocal', 'not', 'or', +'pass', 'raise', 'return', 'try', 'while', 'with', 'yield'] ``` as well as soft keywords: -``` - >>> import keyword - >>> keyword.softkwlist - ['_', 'case', 'match'] +```pycon +>>> import keyword +>>> keyword.softkwlist +['_', 'case', 'match'] ``` > [!CAUTION] @@ -736,7 +735,7 @@ displayed when the error is reported. > rule or not. For example: ``` - $ 42 + $ 42 ``` should trigger the syntax error in the `$` character. If your rule is not correctly defined this @@ -744,7 +743,7 @@ won't happen. As another example, suppose that you try to define a rule to match `print` statements in order to create a better error message and you define it as: ``` - invalid_print: "print" expression +invalid_print: "print" expression ``` This will **seem** to work because the parser will correctly parse `print(something)` because it is valid @@ -756,7 +755,7 @@ will be reported there instead of the `$` character. Generating AST objects ---------------------- -The output of the C parser used by CPython, which is generated from the +The output of the C parser used by CPython, which is generated from the [grammar file](../Grammar/python.gram), is a Python AST object (using C structures). This means that the actions in the grammar file generate AST objects when they succeed. Constructing these objects can be quite cumbersome @@ -798,7 +797,7 @@ Check the contents of these files to know which is the best place for new tests, depending on the nature of the new feature you are adding. Tests for the parser generator itself can be found in the -[test_peg_generator](../Lib/test_peg_generator) directory. +[test_peg_generator](../Lib/test/test_peg_generator) directory. Debugging generated parsers @@ -816,15 +815,15 @@ For this reason it is a good idea to experiment first by generating a Python parser. To do this, you can go to the [Tools/peg_generator](../Tools/peg_generator) directory on the CPython repository and manually call the parser generator by executing: -``` - $ python -m pegen python +```shell +$ python -m pegen python ``` This will generate a file called `parse.py` in the same directory that you can use to parse some input: -``` - $ python parse.py file_with_source_code_to_test.py +```shell +$ python parse.py file_with_source_code_to_test.py ``` As the generated `parse.py` file is just Python code, you can modify it @@ -848,8 +847,8 @@ can be a bit hard to understand at first. To activate verbose mode you can add the `-d` flag when executing Python: -``` - $ python -d file_to_test.py +```shell +$ python -d file_to_test.py ``` This will print **a lot** of output to `stderr` so it is probably better to dump @@ -857,7 +856,7 @@ it to a file for further analysis. The output consists of trace lines with the following structure:: ``` - ('>'|'-'|'+'|'!') []: ... + ('>'|'-'|'+'|'!') []: ... ``` Every line is indented by a different amount (``) depending on how diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index e0d20632516142..26a5197c6e70f3 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -2,6 +2,7 @@ *Interned* strings are conceptually part of an interpreter-global *set* of interned strings, meaning that: + - no two interned strings have the same content (across an interpreter); - two interned strings can be safely compared using pointer equality (Python `is`). @@ -61,6 +62,7 @@ if it's interned and mortal it needs extra processing in The converse is not true: interned strings can be mortal. For mortal interned strings: + - the 2 references from the interned dict (key & value) are excluded from their refcount - the deallocator (`unicode_dealloc`) removes the string from the interned dict @@ -90,6 +92,7 @@ modify in place. The functions take ownership of (“steal”) the reference to their argument, and update the argument with a *new* reference. This means: + - They're “reference neutral”. - They must not be called with a borrowed reference. diff --git a/Lib/_android_support.py b/Lib/_android_support.py index 7572745c851847..ae506f6a4b57b8 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -6,7 +6,7 @@ # The maximum length of a log message in bytes, including the level marker and # tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at # https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71. -# Messages longer than this will be be truncated by logcat. This limit has already +# Messages longer than this will be truncated by logcat. This limit has already # been reduced at least once in the history of Android (from 4076 to 4068 between # API level 23 and 26), so leave some headroom. MAX_BYTES_PER_WRITE = 4000 diff --git a/Lib/_apple_support.py b/Lib/_apple_support.py new file mode 100644 index 00000000000000..92febdcf587070 --- /dev/null +++ b/Lib/_apple_support.py @@ -0,0 +1,66 @@ +import io +import sys + + +def init_streams(log_write, stdout_level, stderr_level): + # Redirect stdout and stderr to the Apple system log. This method is + # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger + # is enabled. + sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors) + sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors) + + +class SystemLog(io.TextIOWrapper): + def __init__(self, log_write, level, **kwargs): + kwargs.setdefault("encoding", "UTF-8") + kwargs.setdefault("line_buffering", True) + super().__init__(LogStream(log_write, level), **kwargs) + + def __repr__(self): + return f"" + + def write(self, s): + if not isinstance(s, str): + raise TypeError( + f"write() argument must be str, not {type(s).__name__}") + + # In case `s` is a str subclass that writes itself to stdout or stderr + # when we call its methods, convert it to an actual str. + s = str.__str__(s) + + # We want to emit one log message per line, so split + # the string before sending it to the superclass. + for line in s.splitlines(keepends=True): + super().write(line) + + return len(s) + + +class LogStream(io.RawIOBase): + def __init__(self, log_write, level): + self.log_write = log_write + self.level = level + + def __repr__(self): + return f"" + + def writable(self): + return True + + def write(self, b): + if type(b) is not bytes: + try: + b = bytes(memoryview(b)) + except TypeError: + raise TypeError( + f"write() argument must be bytes-like, not {type(b).__name__}" + ) from None + + # Writing an empty string to the stream should have no effect. + if b: + # Encode null bytes using "modified UTF-8" to avoid truncating the + # message. This should not affect the return value, as the caller + # may be expecting it to match the length of the input. + self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80")) + + return len(b) diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 5b60570c6c592a..ec036199331396 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -97,7 +97,7 @@ class DecimalException(ArithmeticError): Used exceptions derive from this. If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only called if the others are present. This isn't actually used for anything, though. @@ -145,7 +145,7 @@ class InvalidOperation(DecimalException): x ** (+-)INF An operand is invalid - The result of the operation after these is a quiet positive NaN, + The result of the operation after this is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. diff --git a/Lib/_pyio.py b/Lib/_pyio.py index 42b0aea4e2eb2e..14961c39d3541d 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -2545,9 +2545,12 @@ def read(self, size=None): size = size_index() decoder = self._decoder or self._get_decoder() if size < 0: + chunk = self.buffer.read() + if chunk is None: + raise BlockingIOError("Read returned None.") # Read everything. result = (self._get_decoded_chars() + - decoder.decode(self.buffer.read(), final=True)) + decoder.decode(chunk, final=True)) if self._snapshot is not None: self._set_decoded_chars('') self._snapshot = None diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 5f4d2475c0169b..e6e23596db6f99 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -301,8 +301,6 @@ def __init__(self, locale_time=None): 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", - #XXX: Does 'Y' need to worry about having less or more than - # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index f94bf10b4225e7..f1ab9b12d69a5d 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1175,6 +1175,7 @@ def writelines(self, list_of_data): # If the entire buffer couldn't be written, register a write handler if self._buffer: self._loop._add_writer(self._sock_fd, self._write_ready) + self._maybe_pause_protocol() def can_write_eof(self): return True diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d688141f9b183d..78229ac54b80da 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -59,6 +59,8 @@ except ImportError: pass +heapq = None # Lazily imported + ################################################################################ ### OrderedDict @@ -633,7 +635,10 @@ def most_common(self, n=None): return sorted(self.items(), key=_itemgetter(1), reverse=True) # Lazy import to speedup Python startup time - import heapq + global heapq + if heapq is None: + import heapq + return heapq.nlargest(n, self.items(), key=_itemgetter(1)) def elements(self): diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index ac6493892068e9..2f2b0ca9f38633 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -19,7 +19,7 @@ raise Exception("Version number mismatch", __version__, _ctypes_version) if _os.name == "nt": - from _ctypes import COMError, FormatError + from _ctypes import COMError, CopyComPointer, FormatError DEFAULT_MODE = RTLD_LOCAL if _os.name == "posix" and _sys.platform == "darwin": diff --git a/Lib/dis.py b/Lib/dis.py index 1718e39cceb638..aa22404c6687e1 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1115,7 +1115,7 @@ def dis(self): return output.getvalue() -def main(): +def main(args=None): import argparse parser = argparse.ArgumentParser() @@ -1125,8 +1125,10 @@ def main(): help='show instruction offsets') parser.add_argument('-P', '--show-positions', action='store_true', help='show instruction positions') + parser.add_argument('-S', '--specialized', action='store_true', + help='show specialized bytecode') parser.add_argument('infile', nargs='?', default='-') - args = parser.parse_args() + args = parser.parse_args(args=args) if args.infile == '-': name = '' source = sys.stdin.buffer.read() @@ -1135,7 +1137,8 @@ def main(): with open(args.infile, 'rb') as infile: source = infile.read() code = compile(source, name, "exec") - dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets, show_positions=args.show_positions) + dis(code, show_caches=args.show_caches, adaptive=args.specialized, + show_offsets=args.show_offsets, show_positions=args.show_positions) if __name__ == "__main__": main() diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index d7e8d08b2d92c1..23d5461f86fc23 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -425,9 +425,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' diff --git a/Lib/inspect.py b/Lib/inspect.py index e3f74e9f047eaf..b7d8271f8a471f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -2943,11 +2943,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params = OrderedDict() top_kind = _POSITIONAL_ONLY seen_default = False + seen_var_parameters = set() for param in parameters: kind = param.kind name = param.name + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + if kind in seen_var_parameters: + msg = f'more than one {kind.description} parameter' + raise ValueError(msg) + + seen_var_parameters.add(kind) + if kind < top_kind: msg = ( 'wrong parameter order: {} parameter before {} ' diff --git a/Lib/linecache.py b/Lib/linecache.py index 4b38a0464d8747..8ba2df73d5a8fb 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -49,14 +49,17 @@ def checkcache(filename=None): (This is not checked upon each call!)""" if filename is None: - filenames = list(cache.keys()) - elif filename in cache: - filenames = [filename] + # get keys atomically + filenames = cache.copy().keys() else: - return + filenames = [filename] for filename in filenames: - entry = cache[filename] + try: + entry = cache[filename] + except KeyError: + continue + if len(entry) == 1: # lazy cache entry, leave it lazy. continue diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 996887cb713942..710aba9685efda 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -963,7 +963,7 @@ def answer_challenge(connection, authkey: bytes): f'Protocol error, expected challenge: {message=}') message = message[len(_CHALLENGE):] if len(message) < _MD5ONLY_MESSAGE_LENGTH: - raise AuthenticationError('challenge too short: {len(message)} bytes') + raise AuthenticationError(f'challenge too short: {len(message)} bytes') digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 5da3acd31997e5..ec1bac9ef49350 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -5,8 +5,6 @@ operating systems. """ -from pathlib._abc import * from pathlib._local import * -__all__ = (_abc.__all__ + - _local.__all__) +__all__ = _local.__all__ diff --git a/Lib/pathlib/_abc.py b/Lib/pathlib/_abc.py index 2c243d470d4eda..b4560295300c28 100644 --- a/Lib/pathlib/_abc.py +++ b/Lib/pathlib/_abc.py @@ -14,79 +14,17 @@ import functools import operator import posixpath -from errno import EINVAL, EXDEV +from errno import EINVAL from glob import _GlobberBase, _no_recurse_symlinks -from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO +from stat import S_ISDIR, S_ISLNK, S_ISREG from pathlib._os import copyfileobj -__all__ = ["UnsupportedOperation"] - - -class UnsupportedOperation(NotImplementedError): - """An exception that is raised when an unsupported operation is attempted. - """ - pass - - @functools.cache def _is_case_sensitive(parser): return parser.normcase('Aa') == 'Aa' - -class ParserBase: - """Base class for path parsers, which do low-level path manipulation. - - Path parsers provide a subset of the os.path API, specifically those - functions needed to provide PurePathBase functionality. Each PurePathBase - subclass references its path parser via a 'parser' class attribute. - - Every method in this base class raises an UnsupportedOperation exception. - """ - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - - @property - def sep(self): - """The character used to separate path components.""" - raise UnsupportedOperation(self._unsupported_msg('sep')) - - def join(self, path, *paths): - """Join path segments.""" - raise UnsupportedOperation(self._unsupported_msg('join()')) - - def split(self, path): - """Split the path into a pair (head, tail), where *head* is everything - before the final path separator, and *tail* is everything after. - Either part may be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('split()')) - - def splitdrive(self, path): - """Split the path into a 2-item tuple (drive, tail), where *drive* is - a device name or mount point, and *tail* is everything after the - drive. Either part may be empty.""" - raise UnsupportedOperation(self._unsupported_msg('splitdrive()')) - - def splitext(self, path): - """Split the path into a pair (root, ext), where *ext* is empty or - begins with a period and contains at most one period, - and *root* is everything before the extension.""" - raise UnsupportedOperation(self._unsupported_msg('splitext()')) - - def normcase(self, path): - """Normalize the case of the path.""" - raise UnsupportedOperation(self._unsupported_msg('normcase()')) - - def isabs(self, path): - """Returns whether the path is absolute, i.e. unaffected by the - current directory or drive.""" - raise UnsupportedOperation(self._unsupported_msg('isabs()')) - - class PathGlobber(_GlobberBase): """ Class providing shell-style globbing for path objects. @@ -94,7 +32,7 @@ class PathGlobber(_GlobberBase): lexists = operator.methodcaller('exists', follow_symlinks=False) add_slash = operator.methodcaller('joinpath', '') - scandir = operator.methodcaller('scandir') + scandir = operator.methodcaller('_scandir') @staticmethod def concat_path(path, text): @@ -115,13 +53,8 @@ class PurePathBase: # The `_raw_paths` slot stores unjoined string paths. This is set in # the `__init__()` method. '_raw_paths', - - # The '_resolving' slot stores a boolean indicating whether the path - # is being processed by `PathBase.resolve()`. This prevents duplicate - # work from occurring when `resolve()` calls `stat()` or `readlink()`. - '_resolving', ) - parser = ParserBase() + parser = posixpath _globber = PathGlobber def __init__(self, *args): @@ -130,7 +63,6 @@ def __init__(self, *args): raise TypeError( f"argument should be a str, not {type(arg).__name__!r}") self._raw_paths = list(args) - self._resolving = False def with_segments(self, *pathsegments): """Construct a new path object from any number of path-like objects. @@ -339,9 +271,7 @@ def parent(self): path = str(self) parent = self.parser.split(path)[0] if path != parent: - parent = self.with_segments(parent) - parent._resolving = self._resolving - return parent + return self.with_segments(parent) return self @property @@ -414,8 +344,8 @@ class PathBase(PurePathBase): This class provides dummy implementations for many methods that derived classes can override selectively; the default implementations raise - UnsupportedOperation. The most basic methods, such as stat() and open(), - directly raise UnsupportedOperation; these basic methods are called by + NotImplementedError. The most basic methods, such as stat() and open(), + directly raise NotImplementedError; these basic methods are called by other methods such as is_dir() and read_text(). The Path class derives this class to implement local filesystem paths. @@ -424,27 +354,12 @@ class PathBase(PurePathBase): """ __slots__ = () - # Maximum number of symlinks to follow in resolve() - _max_symlinks = 40 - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported" - def stat(self, *, follow_symlinks=True): """ Return the result of the stat() system call on this path, like os.stat() does. """ - raise UnsupportedOperation(self._unsupported_msg('stat()')) - - def lstat(self): - """ - Like stat(), except if the path points to a symlink, the symlink's - status information is returned, rather than its target's. - """ - return self.stat(follow_symlinks=False) - + raise NotImplementedError # Convenience functions for querying the stat results @@ -480,105 +395,20 @@ def is_file(self, *, follow_symlinks=True): except (OSError, ValueError): return False - def is_mount(self): - """ - Check if this path is a mount point - """ - # Need to exist and be a dir - if not self.exists() or not self.is_dir(): - return False - - try: - parent_dev = self.parent.stat().st_dev - except OSError: - return False - - dev = self.stat().st_dev - if dev != parent_dev: - return True - ino = self.stat().st_ino - parent_ino = self.parent.stat().st_ino - return ino == parent_ino - def is_symlink(self): """ Whether this path is a symbolic link. """ try: - return S_ISLNK(self.lstat().st_mode) - except (OSError, ValueError): - return False - - def is_junction(self): - """ - Whether this path is a junction. - """ - # Junctions are a Windows-only feature, not present in POSIX nor the - # majority of virtual filesystems. There is no cross-platform idiom - # to check for junctions (using stat().st_mode). - return False - - def is_block_device(self): - """ - Whether this path is a block device. - """ - try: - return S_ISBLK(self.stat().st_mode) - except (OSError, ValueError): - return False - - def is_char_device(self): - """ - Whether this path is a character device. - """ - try: - return S_ISCHR(self.stat().st_mode) + return S_ISLNK(self.stat(follow_symlinks=False).st_mode) except (OSError, ValueError): return False - def is_fifo(self): - """ - Whether this path is a FIFO. - """ - try: - return S_ISFIFO(self.stat().st_mode) - except (OSError, ValueError): - return False - - def is_socket(self): - """ - Whether this path is a socket. - """ - try: - return S_ISSOCK(self.stat().st_mode) - except (OSError, ValueError): - return False - - def samefile(self, other_path): - """Return whether other_path is the same or not as this file - (as returned by os.path.samefile()). - """ - st = self.stat() - try: - other_st = other_path.stat() - except AttributeError: - other_st = self.with_segments(other_path).stat() - return (st.st_ino == other_st.st_ino and - st.st_dev == other_st.st_dev) - def _ensure_different_file(self, other_path): """ Raise OSError(EINVAL) if both paths refer to the same file. """ - try: - if not self.samefile(other_path): - return - except (OSError, ValueError): - return - err = OSError(EINVAL, "Source and target are the same file") - err.filename = str(self) - err.filename2 = str(other_path) - raise err + pass def _ensure_distinct_path(self, other_path): """ @@ -605,7 +435,7 @@ def open(self, mode='r', buffering=-1, encoding=None, Open the file pointed to by this path and return a file object, as the built-in open() function does. """ - raise UnsupportedOperation(self._unsupported_msg('open()')) + raise NotImplementedError def read_bytes(self): """ @@ -640,13 +470,14 @@ def write_text(self, data, encoding=None, errors=None, newline=None): with self.open(mode='w', encoding=encoding, errors=errors, newline=newline) as f: return f.write(data) - def scandir(self): - """Yield os.DirEntry objects of the directory contents. + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - raise UnsupportedOperation(self._unsupported_msg('scandir()')) + import contextlib + return contextlib.nullcontext(self.iterdir()) def iterdir(self): """Yield path objects of the directory contents. @@ -654,9 +485,7 @@ def iterdir(self): The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. """ - with self.scandir() as entries: - names = [entry.name for entry in entries] - return map(self.joinpath, names) + raise NotImplementedError def _glob_selector(self, parts, case_sensitive, recurse_symlinks): if case_sensitive is None: @@ -706,7 +535,7 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): if not top_down: paths.append((path, dirnames, filenames)) try: - with path.scandir() as entries: + with path._scandir() as entries: for entry in entries: name = entry.name try: @@ -729,89 +558,18 @@ def walk(self, top_down=True, on_error=None, follow_symlinks=False): yield path, dirnames, filenames paths += [path.joinpath(d) for d in reversed(dirnames)] - def absolute(self): - """Return an absolute version of this path - No normalization or symlink resolution is performed. - - Use resolve() to resolve symlinks and remove '..' segments. - """ - if self.is_absolute(): - return self - elif self.parser is not posixpath: - raise UnsupportedOperation(self._unsupported_msg('absolute()')) - else: - # Treat the root directory as the current working directory. - return self.with_segments('/', *self._raw_paths) - - @classmethod - def cwd(cls): - """Return a new path pointing to the current working directory.""" - # We call 'absolute()' rather than using 'os.getcwd()' directly to - # enable users to replace the implementation of 'absolute()' in a - # subclass and benefit from the new behaviour here. This works because - # os.path.abspath('.') == os.getcwd(). - return cls().absolute() - - def expanduser(self): - """ Return a new path with expanded ~ and ~user constructs - (as returned by os.path.expanduser) - """ - raise UnsupportedOperation(self._unsupported_msg('expanduser()')) - - @classmethod - def home(cls): - """Return a new path pointing to expanduser('~'). - """ - return cls("~").expanduser() - def readlink(self): """ Return the path to which the symbolic link points. """ - raise UnsupportedOperation(self._unsupported_msg('readlink()')) - readlink._supported = False - - def resolve(self, strict=False): - """ - Make the path absolute, resolving all symlinks on the way and also - normalizing it. - """ - if self._resolving: - return self - elif self.parser is not posixpath: - raise UnsupportedOperation(self._unsupported_msg('resolve()')) - - def raise_error(*args): - raise OSError("Unsupported operation.") - - getcwd = raise_error - if strict or getattr(self.readlink, '_supported', True): - def lstat(path_str): - path = self.with_segments(path_str) - path._resolving = True - return path.lstat() - - def readlink(path_str): - path = self.with_segments(path_str) - path._resolving = True - return str(path.readlink()) - else: - # If the user has *not* overridden the `readlink()` method, then - # symlinks are unsupported and (in non-strict mode) we can improve - # performance by not calling `path.lstat()`. - lstat = readlink = raise_error - - return self.with_segments(posixpath._realpath( - str(self.absolute()), strict, self.parser.sep, - getcwd=getcwd, lstat=lstat, readlink=readlink, - maxlinks=self._max_symlinks)) + raise NotImplementedError def symlink_to(self, target, target_is_directory=False): """ Make this path a symlink pointing to the target path. Note the order of arguments (link, target) is the reverse of os.symlink. """ - raise UnsupportedOperation(self._unsupported_msg('symlink_to()')) + raise NotImplementedError def _symlink_to_target_of(self, link): """ @@ -820,25 +578,11 @@ def _symlink_to_target_of(self, link): """ self.symlink_to(link.readlink()) - def hardlink_to(self, target): - """ - Make this path a hard link pointing to the same file as *target*. - - Note the order of arguments (self, target) is the reverse of os.link's. - """ - raise UnsupportedOperation(self._unsupported_msg('hardlink_to()')) - - def touch(self, mode=0o666, exist_ok=True): - """ - Create this file with the given access mode, if it doesn't exist. - """ - raise UnsupportedOperation(self._unsupported_msg('touch()')) - def mkdir(self, mode=0o777, parents=False, exist_ok=False): """ Create a new directory at this given path. """ - raise UnsupportedOperation(self._unsupported_msg('mkdir()')) + raise NotImplementedError # Metadata keys supported by this path type. _readable_metadata = _writable_metadata = frozenset() @@ -847,13 +591,13 @@ def _read_metadata(self, keys=None, *, follow_symlinks=True): """ Returns path metadata as a dict with string keys. """ - raise UnsupportedOperation(self._unsupported_msg('_read_metadata()')) + raise NotImplementedError def _write_metadata(self, metadata, *, follow_symlinks=True): """ Sets path metadata from the given dict with string keys. """ - raise UnsupportedOperation(self._unsupported_msg('_write_metadata()')) + raise NotImplementedError def _copy_metadata(self, target, *, follow_symlinks=True): """ @@ -926,45 +670,16 @@ def copy_into(self, target_dir, *, follow_symlinks=True, dirs_exist_ok=dirs_exist_ok, preserve_metadata=preserve_metadata) - def rename(self, target): - """ - Rename this path to the target path. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. - """ - raise UnsupportedOperation(self._unsupported_msg('rename()')) - - def replace(self, target): + def _delete(self): """ - Rename this path to the target path, overwriting if that path exists. - - The target path may be absolute or relative. Relative paths are - interpreted relative to the current working directory, *not* the - directory of the Path object. - - Returns the new Path instance pointing to the target path. + Delete this file or directory (including all sub-directories). """ - raise UnsupportedOperation(self._unsupported_msg('replace()')) + raise NotImplementedError def move(self, target): """ Recursively move this file or directory tree to the given destination. """ - self._ensure_different_file(target) - try: - return self.replace(target) - except UnsupportedOperation: - pass - except TypeError: - if not isinstance(target, PathBase): - raise - except OSError as err: - if err.errno != EXDEV: - raise target = self.copy(target, follow_symlinks=False, preserve_metadata=True) self._delete() return target @@ -981,74 +696,3 @@ def move_into(self, target_dir): else: target = self.with_segments(target_dir, name) return self.move(target) - - def chmod(self, mode, *, follow_symlinks=True): - """ - Change the permissions of the path, like os.chmod(). - """ - raise UnsupportedOperation(self._unsupported_msg('chmod()')) - - def lchmod(self, mode): - """ - Like chmod(), except if the path points to a symlink, the symlink's - permissions are changed, rather than its target's. - """ - self.chmod(mode, follow_symlinks=False) - - def unlink(self, missing_ok=False): - """ - Remove this file or link. - If the path is a directory, use rmdir() instead. - """ - raise UnsupportedOperation(self._unsupported_msg('unlink()')) - - def rmdir(self): - """ - Remove this directory. The directory must be empty. - """ - raise UnsupportedOperation(self._unsupported_msg('rmdir()')) - - def _delete(self): - """ - Delete this file or directory (including all sub-directories). - """ - if self.is_symlink() or self.is_junction(): - self.unlink() - elif self.is_dir(): - self._rmtree() - else: - self.unlink() - - def _rmtree(self): - def on_error(err): - raise err - results = self.walk( - on_error=on_error, - top_down=False, # So we rmdir() empty directories. - follow_symlinks=False) - for dirpath, _, filenames in results: - for filename in filenames: - filepath = dirpath / filename - filepath.unlink() - dirpath.rmdir() - - def owner(self, *, follow_symlinks=True): - """ - Return the login name of the file owner. - """ - raise UnsupportedOperation(self._unsupported_msg('owner()')) - - def group(self, *, follow_symlinks=True): - """ - Return the group name of the file gid. - """ - raise UnsupportedOperation(self._unsupported_msg('group()')) - - @classmethod - def from_uri(cls, uri): - """Return a new path from the given 'file' URI.""" - raise UnsupportedOperation(cls._unsupported_msg('from_uri()')) - - def as_uri(self): - """Return the path as a URI.""" - raise UnsupportedOperation(self._unsupported_msg('as_uri()')) diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py index b27f456d375225..b933dd512eeb28 100644 --- a/Lib/pathlib/_local.py +++ b/Lib/pathlib/_local.py @@ -4,8 +4,10 @@ import os import posixpath import sys +from errno import EINVAL, EXDEV from glob import _StringGlobber from itertools import chain +from stat import S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO from _collections_abc import Sequence try: @@ -19,15 +21,22 @@ from pathlib._os import (copyfile, file_metadata_keys, read_file_metadata, write_file_metadata) -from pathlib._abc import UnsupportedOperation, PurePathBase, PathBase +from pathlib._abc import PurePathBase, PathBase __all__ = [ + "UnsupportedOperation", "PurePath", "PurePosixPath", "PureWindowsPath", "Path", "PosixPath", "WindowsPath", ] +class UnsupportedOperation(NotImplementedError): + """An exception that is raised when an unsupported operation is attempted. + """ + pass + + class _PathParents(Sequence): """This object provides sequence-like access to the logical ancestors of a path. Don't try to construct it yourself.""" @@ -524,11 +533,6 @@ class Path(PathBase, PurePath): but cannot instantiate a WindowsPath on a POSIX system or vice versa. """ __slots__ = () - as_uri = PurePath.as_uri - - @classmethod - def _unsupported_msg(cls, attribute): - return f"{cls.__name__}.{attribute} is unsupported on this system" def __new__(cls, *args, **kwargs): if cls is Path: @@ -542,6 +546,13 @@ def stat(self, *, follow_symlinks=True): """ return os.stat(self, follow_symlinks=follow_symlinks) + def lstat(self): + """ + Like stat(), except if the path points to a symlink, the symlink's + status information is returned, rather than its target's. + """ + return os.lstat(self) + def exists(self, *, follow_symlinks=True): """ Whether this path exists. @@ -588,6 +599,68 @@ def is_junction(self): """ return os.path.isjunction(self) + def is_block_device(self): + """ + Whether this path is a block device. + """ + try: + return S_ISBLK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_char_device(self): + """ + Whether this path is a character device. + """ + try: + return S_ISCHR(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_fifo(self): + """ + Whether this path is a FIFO. + """ + try: + return S_ISFIFO(self.stat().st_mode) + except (OSError, ValueError): + return False + + def is_socket(self): + """ + Whether this path is a socket. + """ + try: + return S_ISSOCK(self.stat().st_mode) + except (OSError, ValueError): + return False + + def samefile(self, other_path): + """Return whether other_path is the same or not as this file + (as returned by os.path.samefile()). + """ + st = self.stat() + try: + other_st = other_path.stat() + except AttributeError: + other_st = self.with_segments(other_path).stat() + return (st.st_ino == other_st.st_ino and + st.st_dev == other_st.st_dev) + + def _ensure_different_file(self, other_path): + """ + Raise OSError(EINVAL) if both paths refer to the same file. + """ + try: + if not self.samefile(other_path): + return + except (OSError, ValueError): + return + err = OSError(EINVAL, "Source and target are the same file") + err.filename = str(self) + err.filename2 = str(other_path) + raise err + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ @@ -627,8 +700,8 @@ def _filter_trailing_slash(self, paths): path_str = path_str[:-1] yield path_str - def scandir(self): - """Yield os.DirEntry objects of the directory contents. + def _scandir(self): + """Yield os.DirEntry-like objects of the directory contents. The children are yielded in arbitrary order, and the special entries '.' and '..' are not included. @@ -719,6 +792,14 @@ def absolute(self): tail.extend(self._tail) return self._from_parsed_parts(drive, root, tail) + @classmethod + def cwd(cls): + """Return a new path pointing to the current working directory.""" + cwd = os.getcwd() + path = cls(cwd) + path._str = cwd # getcwd() returns a normalized path + return path + def resolve(self, strict=False): """ Make the path absolute, resolving all symlinks on the way and also @@ -734,6 +815,13 @@ def owner(self, *, follow_symlinks=True): """ uid = self.stat(follow_symlinks=follow_symlinks).st_uid return pwd.getpwuid(uid).pw_name + else: + def owner(self, *, follow_symlinks=True): + """ + Return the login name of the file owner. + """ + f = f"{type(self).__name__}.owner()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if grp: def group(self, *, follow_symlinks=True): @@ -742,6 +830,13 @@ def group(self, *, follow_symlinks=True): """ gid = self.stat(follow_symlinks=follow_symlinks).st_gid return grp.getgrgid(gid).gr_name + else: + def group(self, *, follow_symlinks=True): + """ + Return the group name of the file gid. + """ + f = f"{type(self).__name__}.group()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if hasattr(os, "readlink"): def readlink(self): @@ -749,6 +844,13 @@ def readlink(self): Return the path to which the symbolic link points. """ return self.with_segments(os.readlink(self)) + else: + def readlink(self): + """ + Return the path to which the symbolic link points. + """ + f = f"{type(self).__name__}.readlink()" + raise UnsupportedOperation(f"{f} is unsupported on this system") def touch(self, mode=0o666, exist_ok=True): """ @@ -813,6 +915,13 @@ def chmod(self, mode, *, follow_symlinks=True): """ os.chmod(self, mode, follow_symlinks=follow_symlinks) + def lchmod(self, mode): + """ + Like chmod(), except if the path points to a symlink, the symlink's + permissions are changed, rather than its target's. + """ + self.chmod(mode, follow_symlinks=False) + def unlink(self, missing_ok=False): """ Remove this file or link. @@ -830,10 +939,18 @@ def rmdir(self): """ os.rmdir(self) - def _rmtree(self): - # Lazy import to improve module import time - import shutil - shutil.rmtree(self) + def _delete(self): + """ + Delete this file or directory (including all sub-directories). + """ + if self.is_symlink() or self.is_junction(): + self.unlink() + elif self.is_dir(): + # Lazy import to improve module import time + import shutil + shutil.rmtree(self) + else: + self.unlink() def rename(self, target): """ @@ -861,6 +978,22 @@ def replace(self, target): os.replace(self, target) return self.with_segments(target) + def move(self, target): + """ + Recursively move this file or directory tree to the given destination. + """ + self._ensure_different_file(target) + try: + return self.replace(target) + except TypeError: + if not isinstance(target, PathBase): + raise + except OSError as err: + if err.errno != EXDEV: + raise + # Fall back to copy+delete. + return PathBase.move(self, target) + if hasattr(os, "symlink"): def symlink_to(self, target, target_is_directory=False): """ @@ -868,6 +1001,14 @@ def symlink_to(self, target, target_is_directory=False): Note the order of arguments (link, target) is the reverse of os.symlink. """ os.symlink(target, self, target_is_directory) + else: + def symlink_to(self, target, target_is_directory=False): + """ + Make this path a symlink pointing to the target path. + Note the order of arguments (link, target) is the reverse of os.symlink. + """ + f = f"{type(self).__name__}.symlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") if os.name == 'nt': def _symlink_to_target_of(self, link): @@ -885,6 +1026,15 @@ def hardlink_to(self, target): Note the order of arguments (self, target) is the reverse of os.link's. """ os.link(target, self) + else: + def hardlink_to(self, target): + """ + Make this path a hard link pointing to the same file as *target*. + + Note the order of arguments (self, target) is the reverse of os.link's. + """ + f = f"{type(self).__name__}.hardlink_to()" + raise UnsupportedOperation(f"{f} is unsupported on this system") def expanduser(self): """ Return a new path with expanded ~ and ~user constructs @@ -900,6 +1050,15 @@ def expanduser(self): return self + @classmethod + def home(cls): + """Return a new path pointing to expanduser('~'). + """ + homedir = os.path.expanduser("~") + if homedir == "~": + raise RuntimeError("Could not determine home directory.") + return cls(homedir) + @classmethod def from_uri(cls, uri): """Return a new path from the given 'file' URI.""" diff --git a/Lib/pathlib/_types.py b/Lib/pathlib/_types.py new file mode 100644 index 00000000000000..60df94d0b46049 --- /dev/null +++ b/Lib/pathlib/_types.py @@ -0,0 +1,22 @@ +""" +Protocols for supporting classes in pathlib. +""" +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class Parser(Protocol): + """Protocol for path parsers, which do low-level path manipulation. + + Path parsers provide a subset of the os.path API, specifically those + functions needed to provide PurePathBase functionality. Each PurePathBase + subclass references its path parser via a 'parser' class attribute. + """ + + sep: str + def join(self, path: str, *paths: str) -> str: ... + def split(self, path: str) -> tuple[str, str]: ... + def splitdrive(self, path: str) -> tuple[str, str]: ... + def splitext(self, path: str) -> tuple[str, str]: ... + def normcase(self, path: str) -> str: ... + def isabs(self, path: str) -> bool: ... diff --git a/Lib/pdb.py b/Lib/pdb.py index b7f6fd4323407e..10d1923cdad2d6 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -438,6 +438,13 @@ def user_line(self, frame): if (self.mainpyfile != self.canonic(frame.f_code.co_filename)): return self._wait_for_mainpyfile = False + if self.trace_opcodes: + # GH-127321 + # We want to avoid stopping at an opcode that does not have + # an associated line number because pdb does not like it + if frame.f_lineno is None: + self.set_stepinstr() + return self.bp_commands(frame) self.interaction(frame, None) diff --git a/Lib/pickle.py b/Lib/pickle.py index 25dadb3f75a573..1920973e3f83e9 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -1387,7 +1387,7 @@ def load_int(self): elif data == TRUE[1:]: val = True else: - val = int(data, 0) + val = int(data) self.append(val) dispatch[INT[0]] = load_int @@ -1407,7 +1407,7 @@ def load_long(self): val = self.readline()[:-1] if val and val[-1] == b'L'[0]: val = val[:-1] - self.append(int(val, 0)) + self.append(int(val)) dispatch[LONG[0]] = load_long def load_long1(self): diff --git a/Lib/platform.py b/Lib/platform.py index 239e660cd1621d..1f6baed66d3df9 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -353,7 +353,8 @@ def _wmi_query(table, *keys): ] _WIN32_SERVER_RELEASES = [ - ((10, 1, 0), "post2022Server"), + ((10, 1, 0), "post2025Server"), + ((10, 0, 26100), "2025Server"), ((10, 0, 20348), "2022Server"), ((10, 0, 17763), "2019Server"), ((6, 4, 0), "2016Server"), diff --git a/Lib/poplib.py b/Lib/poplib.py index 1a1629d175b6d9..beb93a0d57cf93 100644 --- a/Lib/poplib.py +++ b/Lib/poplib.py @@ -309,7 +309,7 @@ def close(self): # optional commands: def rpop(self, user): - """Not sure what this does.""" + """Send RPOP command to access the mailbox with an alternate user.""" return self._shortcmd('RPOP %s' % user) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ee52700b51fd07..ed7b6a335d01d4 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -173,9 +173,7 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) # Mutex guarding initialization of _CONFIG_VARS. _CONFIG_VARS_LOCK = threading.RLock() @@ -318,14 +316,22 @@ def get_default_scheme(): def get_makefile_filename(): """Return the path of the Makefile.""" + + # GH-127429: When cross-compiling, use the Makefile from the target, instead of the host Python. + if cross_base := os.environ.get('_PYTHON_PROJECT_BASE'): + return os.path.join(cross_base, 'Makefile') + if _PYTHON_BUILD: return os.path.join(_PROJECT_BASE, "Makefile") + if hasattr(sys, 'abiflags'): config_dir_name = f'config-{_PY_VERSION_SHORT}{sys.abiflags}' else: config_dir_name = 'config' + if hasattr(sys.implementation, '_multiarch'): config_dir_name += f'-{sys.implementation._multiarch}' + return os.path.join(get_path('stdlib'), config_dir_name, 'Makefile') @@ -464,27 +470,44 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): def _init_config_vars(): global _CONFIG_VARS _CONFIG_VARS = {} + + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + base_prefix = _BASE_PREFIX + base_exec_prefix = _BASE_EXEC_PREFIX + + try: + abiflags = sys.abiflags + except AttributeError: + abiflags = '' + + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + # If we are cross-compiling, load the prefixes from the Makefile instead. + if '_PYTHON_PROJECT_BASE' in os.environ: + prefix = _CONFIG_VARS['prefix'] + exec_prefix = _CONFIG_VARS['exec_prefix'] + base_prefix = _CONFIG_VARS['prefix'] + base_exec_prefix = _CONFIG_VARS['exec_prefix'] + abiflags = _CONFIG_VARS['ABIFLAGS'] + # Normalized versions of prefix and exec_prefix are handy to have; # in fact, these are the standard versions used most places in the # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX + _CONFIG_VARS['prefix'] = prefix + _CONFIG_VARS['exec_prefix'] = exec_prefix _CONFIG_VARS['py_version'] = _PY_VERSION _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['installed_base'] = base_prefix + _CONFIG_VARS['base'] = prefix + _CONFIG_VARS['installed_platbase'] = base_exec_prefix + _CONFIG_VARS['platbase'] = exec_prefix _CONFIG_VARS['projectbase'] = _PROJECT_BASE _CONFIG_VARS['platlibdir'] = sys.platlibdir _CONFIG_VARS['implementation'] = _get_implementation() _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' + _CONFIG_VARS['abiflags'] = abiflags try: _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') except AttributeError: @@ -493,8 +516,6 @@ def _init_config_vars(): if os.name == 'nt': _init_non_posix(_CONFIG_VARS) _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) if _HAS_USER_BASE: # Setting 'userbase' is done below the call to the # init function to enable using 'get_config_var' in @@ -541,9 +562,19 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ + global _CONFIG_VARS_INITIALIZED # Avoid claiming the lock once initialization is complete. - if not _CONFIG_VARS_INITIALIZED: + if _CONFIG_VARS_INITIALIZED: + # GH-126789: If sys.prefix or sys.exec_prefix were updated, invalidate the cache. + prefix = os.path.normpath(sys.prefix) + exec_prefix = os.path.normpath(sys.exec_prefix) + if _CONFIG_VARS['prefix'] != prefix or _CONFIG_VARS['exec_prefix'] != exec_prefix: + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() + else: + # Initialize the config_vars cache. with _CONFIG_VARS_LOCK: # Test again with the lock held to avoid races. Note that # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index 5660a6c5105b9f..10728c709e1811 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -7,6 +7,7 @@ _PYTHON_BUILD, _get_sysconfigdata_name, get_config_h_filename, + get_config_var, get_config_vars, get_default_scheme, get_makefile_filename, @@ -161,7 +162,7 @@ def _print_config_dict(d, stream): def _get_pybuilddir(): pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' - if hasattr(sys, "gettotalrefcount"): + if get_config_var('Py_DEBUG') == '1': pybuilddir += '-pydebug' return pybuilddir @@ -229,11 +230,15 @@ def _generate_posix_vars(): f.write('build_time_vars = ') _print_config_dict(vars, stream=f) + print(f'Written {destfile}') + # Write a JSON file with the output of sysconfig.get_config_vars jsonfile = os.path.join(pybuilddir, _get_json_data_name()) with open(jsonfile, 'w') as f: json.dump(get_config_vars(), f, indent=2) + print(f'Written {jsonfile}') + # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: f.write(pybuilddir) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 4ad0b8b0910bbe..b6ae04ecf2f8ed 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5303,9 +5303,7 @@ Test_meth_coexist_impl(TestObj *self) Test.property [clinic start generated code]*/ -#if defined(Test_property_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -5326,16 +5324,14 @@ Test_property_get(TestObj *self, void *Py_UNUSED(context)) static PyObject * Test_property_get_impl(TestObj *self) -/*[clinic end generated code: output=27b519719d992e03 input=2d92b3449fbc7d2b]*/ +/*[clinic end generated code: output=7cadd0f539805266 input=2d92b3449fbc7d2b]*/ /*[clinic input] @setter Test.property [clinic start generated code]*/ -#if defined(TEST_PROPERTY_HAS_DOCSTR) -# define Test_property_DOCSTR Test_property__doc__ -#else +#if !defined(Test_property_DOCSTR) # define Test_property_DOCSTR NULL #endif #if defined(TEST_PROPERTY_GETSETDEF) @@ -5360,7 +5356,7 @@ Test_property_set(TestObj *self, PyObject *value, void *Py_UNUSED(context)) static int Test_property_set_impl(TestObj *self, PyObject *value) -/*[clinic end generated code: output=d51023f17c4ac3a1 input=3bc3f46a23c83a88]*/ +/*[clinic end generated code: output=e4342fe9bb1d7817 input=3bc3f46a23c83a88]*/ /*[clinic input] output push diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 0c94fcc1907071..bf9a71efbdbff9 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -87,38 +87,40 @@ The argument is a comma-separated list of words indicating the resources to test. Currently only the following are defined: - all - Enable all special resources. + all - Enable all special resources. - none - Disable all special resources (this is the default). + none - Disable all special resources (this is the default). - audio - Tests that use the audio device. (There are known - cases of broken audio drivers that can crash Python or - even the Linux kernel.) + audio - Tests that use the audio device. (There are known + cases of broken audio drivers that can crash Python or + even the Linux kernel.) - curses - Tests that use curses and will modify the terminal's - state and output modes. + curses - Tests that use curses and will modify the terminal's + state and output modes. - largefile - It is okay to run some test that may create huge - files. These tests can take a long time and may - consume >2 GiB of disk space temporarily. + largefile - It is okay to run some test that may create huge + files. These tests can take a long time and may + consume >2 GiB of disk space temporarily. - network - It is okay to run tests that use external network - resource, e.g. testing SSL support for sockets. + extralargefile - Like 'largefile', but even larger (and slower). - decimal - Test the decimal module against a large suite that - verifies compliance with standards. + network - It is okay to run tests that use external network + resource, e.g. testing SSL support for sockets. - cpu - Used for certain CPU-heavy tests. + decimal - Test the decimal module against a large suite that + verifies compliance with standards. - walltime - Long running but not CPU-bound tests. + cpu - Used for certain CPU-heavy tests. - subprocess Run all tests for the subprocess module. + walltime - Long running but not CPU-bound tests. - urlfetch - It is okay to download files required on testing. + subprocess Run all tests for the subprocess module. - gui - Run tests that require a running GUI. + urlfetch - It is okay to download files required on testing. - tzdata - Run tests that require timezone data. + gui - Run tests that require a running GUI. + + tzdata - Run tests that require timezone data. To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 49209b0cec756e..dcbcc6790c68d8 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -6,6 +6,7 @@ import sysconfig import time import trace +from _colorize import get_colors # type: ignore[import-not-found] from typing import NoReturn from test.support import os_helper, MS_WINDOWS, flush_std_streams @@ -270,6 +271,9 @@ def _rerun_failed_tests(self, runtests: RunTests) -> RunTests: return runtests def rerun_failed_tests(self, runtests: RunTests) -> None: + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET + if self.python_cmd: # Temp patch for https://github.com/python/cpython/issues/94052 self.log( @@ -284,7 +288,10 @@ def rerun_failed_tests(self, runtests: RunTests) -> None: rerun_runtests = self._rerun_failed_tests(runtests) if self.results.bad: - print(count(len(self.results.bad), 'test'), "failed again:") + print( + f"{red}{count(len(self.results.bad), 'test')} " + f"failed again:{reset}" + ) printlist(self.results.bad) self.display_result(rerun_runtests) diff --git a/Lib/test/libregrtest/refleak.py b/Lib/test/libregrtest/refleak.py index e783475cc7a36b..d0d1c8cdc9a11b 100644 --- a/Lib/test/libregrtest/refleak.py +++ b/Lib/test/libregrtest/refleak.py @@ -123,9 +123,9 @@ def get_pooled_int(value): xml_filename = 'refleak-xml.tmp' result = None dash_R_cleanup(fs, ps, pic, zdc, abcs) - support.gc_collect() for i in rep_range: + support.gc_collect() current = refleak_helper._hunting_for_refleaks refleak_helper._hunting_for_refleaks = True try: diff --git a/Lib/test/libregrtest/result.py b/Lib/test/libregrtest/result.py index 7553efe5e8abeb..daf7624366ee20 100644 --- a/Lib/test/libregrtest/result.py +++ b/Lib/test/libregrtest/result.py @@ -1,5 +1,6 @@ import dataclasses import json +from _colorize import get_colors # type: ignore[import-not-found] from typing import Any from .utils import ( @@ -105,54 +106,71 @@ def is_failed(self, fail_env_changed: bool) -> bool: return State.is_failed(self.state) def _format_failed(self): + ansi = get_colors() + red, reset = ansi.BOLD_RED, ansi.RESET if self.errors and self.failures: le = len(self.errors) lf = len(self.failures) error_s = "error" + ("s" if le > 1 else "") failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({le} {error_s}, {lf} {failure_s})" + return ( + f"{red}{self.test_name} failed " + f"({le} {error_s}, {lf} {failure_s}){reset}" + ) if self.errors: le = len(self.errors) error_s = "error" + ("s" if le > 1 else "") - return f"{self.test_name} failed ({le} {error_s})" + return f"{red}{self.test_name} failed ({le} {error_s}){reset}" if self.failures: lf = len(self.failures) failure_s = "failure" + ("s" if lf > 1 else "") - return f"{self.test_name} failed ({lf} {failure_s})" + return f"{red}{self.test_name} failed ({lf} {failure_s}){reset}" - return f"{self.test_name} failed" + return f"{red}{self.test_name} failed{reset}" def __str__(self) -> str: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + match self.state: case State.PASSED: - return f"{self.test_name} passed" + return f"{green}{self.test_name} passed{reset}" case State.FAILED: - return self._format_failed() + return f"{red}{self._format_failed()}{reset}" case State.SKIPPED: - return f"{self.test_name} skipped" + return f"{yellow}{self.test_name} skipped{reset}" case State.UNCAUGHT_EXC: - return f"{self.test_name} failed (uncaught exception)" + return ( + f"{red}{self.test_name} failed (uncaught exception){reset}" + ) case State.REFLEAK: - return f"{self.test_name} failed (reference leak)" + return f"{red}{self.test_name} failed (reference leak){reset}" case State.ENV_CHANGED: - return f"{self.test_name} failed (env changed)" + return f"{red}{self.test_name} failed (env changed){reset}" case State.RESOURCE_DENIED: - return f"{self.test_name} skipped (resource denied)" + return f"{yellow}{self.test_name} skipped (resource denied){reset}" case State.INTERRUPTED: - return f"{self.test_name} interrupted" + return f"{yellow}{self.test_name} interrupted{reset}" case State.WORKER_FAILED: - return f"{self.test_name} worker non-zero exit code" + return ( + f"{red}{self.test_name} worker non-zero exit code{reset}" + ) case State.WORKER_BUG: - return f"{self.test_name} worker bug" + return f"{red}{self.test_name} worker bug{reset}" case State.DID_NOT_RUN: - return f"{self.test_name} ran no tests" + return f"{yellow}{self.test_name} ran no tests{reset}" case State.TIMEOUT: assert self.duration is not None, "self.duration is None" return f"{self.test_name} timed out ({format_duration(self.duration)})" case _: - raise ValueError("unknown result state: {state!r}") + raise ValueError( + f"{red}unknown result state: {{state!r}}{reset}" + ) def has_meaningful_duration(self): return State.has_meaningful_duration(self.state) diff --git a/Lib/test/libregrtest/results.py b/Lib/test/libregrtest/results.py index 9eda926966dc7e..a35934fc2c9ca8 100644 --- a/Lib/test/libregrtest/results.py +++ b/Lib/test/libregrtest/results.py @@ -1,5 +1,6 @@ import sys import trace +from _colorize import get_colors # type: ignore[import-not-found] from typing import TYPE_CHECKING from .runtests import RunTests @@ -59,19 +60,24 @@ def no_tests_run(self) -> bool: def get_state(self, fail_env_changed: bool) -> str: state = [] + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW if self.bad: - state.append("FAILURE") + state.append(f"{red}FAILURE{reset}") elif fail_env_changed and self.env_changed: - state.append("ENV CHANGED") + state.append(f"{yellow}ENV CHANGED{reset}") elif self.no_tests_run(): - state.append("NO TESTS RAN") + state.append(f"{yellow}NO TESTS RAN{reset}") if self.interrupted: - state.append("INTERRUPTED") + state.append(f"{yellow}INTERRUPTED{reset}") if self.worker_bug: - state.append("WORKER BUG") + state.append(f"{red}WORKER BUG{reset}") if not state: - state.append("SUCCESS") + state.append(f"{green}SUCCESS{reset}") return ', '.join(state) @@ -197,27 +203,51 @@ def write_junit(self, filename: StrPath) -> None: f.write(s) def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> None: + ansi = get_colors() + green = ansi.GREEN + red = ansi.BOLD_RED + reset = ansi.RESET + yellow = ansi.YELLOW + if print_slowest: self.test_times.sort(reverse=True) print() - print("10 slowest tests:") + print(f"{yellow}10 slowest tests:{reset}") for test_time, test in self.test_times[:10]: - print("- %s: %s" % (test, format_duration(test_time))) + print(f"- {test}: {format_duration(test_time)}") all_tests = [] omitted = set(tests) - self.get_executed() # less important - all_tests.append((sorted(omitted), "test", "{} omitted:")) + all_tests.append( + (sorted(omitted), "test", f"{yellow}{{}} omitted:{reset}") + ) if not quiet: - all_tests.append((self.skipped, "test", "{} skipped:")) - all_tests.append((self.resource_denied, "test", "{} skipped (resource denied):")) - all_tests.append((self.run_no_tests, "test", "{} run no tests:")) + all_tests.append( + (self.skipped, "test", f"{yellow}{{}} skipped:{reset}") + ) + all_tests.append( + ( + self.resource_denied, + "test", + f"{yellow}{{}} skipped (resource denied):{reset}", + ) + ) + all_tests.append( + (self.run_no_tests, "test", f"{yellow}{{}} run no tests:{reset}") + ) # more important - all_tests.append((self.env_changed, "test", "{} altered the execution environment (env changed):")) - all_tests.append((self.rerun, "re-run test", "{}:")) - all_tests.append((self.bad, "test", "{} failed:")) + all_tests.append( + ( + self.env_changed, + "test", + f"{yellow}{{}} altered the execution environment (env changed):{reset}", + ) + ) + all_tests.append((self.rerun, "re-run test", f"{yellow}{{}}:{reset}")) + all_tests.append((self.bad, "test", f"{red}{{}} failed:{reset}")) for tests_list, count_text, title_format in all_tests: if tests_list: @@ -229,26 +259,29 @@ def display_result(self, tests: TestTuple, quiet: bool, print_slowest: bool) -> if self.good and not quiet: print() text = count(len(self.good), "test") - text = f"{text} OK." - if (self.is_all_good() and len(self.good) > 1): + text = f"{green}{text} OK.{reset}" + if self.is_all_good() and len(self.good) > 1: text = f"All {text}" print(text) if self.interrupted: print() - print("Test suite interrupted by signal SIGINT.") + print(f"{yellow}Test suite interrupted by signal SIGINT.{reset}") def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: # Total tests + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW + stats = self.stats text = f'run={stats.tests_run:,}' if filtered: text = f"{text} (filtered)" report = [text] if stats.failures: - report.append(f'failures={stats.failures:,}') + report.append(f'{red}failures={stats.failures:,}{reset}') if stats.skipped: - report.append(f'skipped={stats.skipped:,}') + report.append(f'{yellow}skipped={stats.skipped:,}{reset}') print(f"Total tests: {' '.join(report)}") # Total test files @@ -263,14 +296,14 @@ def display_summary(self, first_runtests: RunTests, filtered: bool) -> None: if filtered: text = f"{text} (filtered)" report = [text] - for name, tests in ( - ('failed', self.bad), - ('env_changed', self.env_changed), - ('skipped', self.skipped), - ('resource_denied', self.resource_denied), - ('rerun', self.rerun), - ('run_no_tests', self.run_no_tests), + for name, tests, color in ( + ('failed', self.bad, red), + ('env_changed', self.env_changed, yellow), + ('skipped', self.skipped, yellow), + ('resource_denied', self.resource_denied, yellow), + ('rerun', self.rerun, yellow), + ('run_no_tests', self.run_no_tests, yellow), ): if tests: - report.append(f'{name}={len(tests)}') + report.append(f'{color}{name}={len(tests)}{reset}') print(f"Total test files: {' '.join(report)}") diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 17323e7f9cf730..0e174f82abed28 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -7,6 +7,7 @@ import traceback import unittest +from _colorize import get_colors # type: ignore[import-not-found] from test import support from test.support import threading_helper @@ -161,6 +162,8 @@ def test_func(): def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, display_failure: bool = True) -> None: # Handle exceptions, detect environment changes. + ansi = get_colors() + red, reset, yellow = ansi.RED, ansi.RESET, ansi.YELLOW # Reset the environment_altered flag to detect if a test altered # the environment @@ -181,18 +184,18 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, _load_run_test(result, runtests) except support.ResourceDenied as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print(f"{yellow}{test_name} skipped -- {exc}{reset}", flush=True) result.state = State.RESOURCE_DENIED return except unittest.SkipTest as exc: if not quiet and not pgo: - print(f"{test_name} skipped -- {exc}", flush=True) + print(f"{yellow}{test_name} skipped -- {exc}{reset}", flush=True) result.state = State.SKIPPED return except support.TestFailedWithDetails as exc: - msg = f"test {test_name} failed" + msg = f"{red}test {test_name} failed{reset}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{red}{msg} -- {exc}{reset}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.errors = exc.errors @@ -200,9 +203,9 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, result.stats = exc.stats return except support.TestFailed as exc: - msg = f"test {test_name} failed" + msg = f"{red}test {test_name} failed{reset}" if display_failure: - msg = f"{msg} -- {exc}" + msg = f"{red}{msg} -- {exc}{reset}" print(msg, file=sys.stderr, flush=True) result.state = State.FAILED result.stats = exc.stats @@ -217,7 +220,7 @@ def _runtest_env_changed_exc(result: TestResult, runtests: RunTests, except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", + print(f"{red}test {test_name} crashed -- {msg}{reset}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC return @@ -300,6 +303,9 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: If runtests.use_junit, xml_data is a list containing each generated testsuite element. """ + ansi = get_colors() + red, reset, yellow = ansi.BOLD_RED, ansi.RESET, ansi.YELLOW + start_time = time.perf_counter() result = TestResult(test_name) pgo = runtests.pgo @@ -308,7 +314,7 @@ def run_single_test(test_name: TestName, runtests: RunTests) -> TestResult: except: if not pgo: msg = traceback.format_exc() - print(f"test {test_name} crashed -- {msg}", + print(f"{red}test {test_name} crashed -- {msg}{reset}", file=sys.stderr, flush=True) result.state = State.UNCAUGHT_EXC diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index 822ac0f4044d9e..00d5779d950e72 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -25,6 +25,7 @@ 'test_threading_local', 'test_threadsignals', 'test_weakref', + 'test_free_threading.test_slots', ] diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index dbc5ef4f9f2cd5..dbd9f27872962d 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit, skip_emscripten_stack_overflow class CommonTest(seq_tests.CommonTest): @@ -59,6 +59,7 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_emscripten_stack_overflow() def test_repr_deep(self): a = self.type2test([]) for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/mapping_tests.py b/Lib/test/mapping_tests.py index ed89a81a6ea685..f249f0021e9c1c 100644 --- a/Lib/test/mapping_tests.py +++ b/Lib/test/mapping_tests.py @@ -1,7 +1,7 @@ # tests common to dict and UserDict import unittest import collections -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class BasicTestMappingProtocol(unittest.TestCase): @@ -622,6 +622,7 @@ def __repr__(self): d = self._full_mapping({1: BadRepr()}) self.assertRaises(Exc, repr, d) + @skip_emscripten_stack_overflow() def test_repr_deep(self): d = self._empty_mapping() for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index cf020a48b81cfa..bdc7ef62943a28 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1012,6 +1012,26 @@ def test_constants(self): self.assertIs(self.loads(b'I01\n.'), True) self.assertIs(self.loads(b'I00\n.'), False) + def test_zero_padded_integers(self): + self.assertEqual(self.loads(b'I010\n.'), 10) + self.assertEqual(self.loads(b'I-010\n.'), -10) + self.assertEqual(self.loads(b'I0010\n.'), 10) + self.assertEqual(self.loads(b'I-0010\n.'), -10) + self.assertEqual(self.loads(b'L010\n.'), 10) + self.assertEqual(self.loads(b'L-010\n.'), -10) + self.assertEqual(self.loads(b'L0010\n.'), 10) + self.assertEqual(self.loads(b'L-0010\n.'), -10) + self.assertEqual(self.loads(b'L010L\n.'), 10) + self.assertEqual(self.loads(b'L-010L\n.'), -10) + + def test_nondecimal_integers(self): + self.assertRaises(ValueError, self.loads, b'I0b10\n.') + self.assertRaises(ValueError, self.loads, b'I0o10\n.') + self.assertRaises(ValueError, self.loads, b'I0x10\n.') + self.assertRaises(ValueError, self.loads, b'L0b10L\n.') + self.assertRaises(ValueError, self.loads, b'L0o10L\n.') + self.assertRaises(ValueError, self.loads, b'L0x10L\n.') + def test_empty_bytestring(self): # issue 11286 empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 2ad267e3e08f0f..5c738ffaa27713 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -535,6 +535,9 @@ def skip_android_selinux(name): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +def skip_emscripten_stack_overflow(): + return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + is_apple_mobile = sys.platform in {"ios", "tvos", "watchos"} is_apple = is_apple_mobile or sys.platform == "darwin" diff --git a/Lib/test/test__locale.py b/Lib/test/test__locale.py index 2c751033ebb3e2..cef84fd9580c37 100644 --- a/Lib/test/test__locale.py +++ b/Lib/test/test__locale.py @@ -102,6 +102,11 @@ def accept(loc): # ps_AF doesn't work on Windows: see bpo-38324 (msg361830) del known_numerics['ps_AF'] +if sys.platform == 'sunos5': + # On Solaris, Japanese ERAs start with the year 1927, + # and thus there's less of them. + known_era['ja_JP'] = (5, '+:1:2019/05/01:2019/12/31:令和:%EC元年') + class _LocaleTests(unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_apple.py b/Lib/test/test_apple.py new file mode 100644 index 00000000000000..ab5296afad1d3f --- /dev/null +++ b/Lib/test/test_apple.py @@ -0,0 +1,155 @@ +import unittest +from _apple_support import SystemLog +from test.support import is_apple +from unittest.mock import Mock, call + +if not is_apple: + raise unittest.SkipTest("Apple-specific") + + +# Test redirection of stdout and stderr to the Apple system log. +class TestAppleSystemLogOutput(unittest.TestCase): + maxDiff = None + + def assert_writes(self, output): + self.assertEqual( + self.log_write.mock_calls, + [ + call(self.log_level, line) + for line in output + ] + ) + + self.log_write.reset_mock() + + def setUp(self): + self.log_write = Mock() + self.log_level = 42 + self.log = SystemLog(self.log_write, self.log_level, errors="replace") + + def test_repr(self): + self.assertEqual(repr(self.log), "") + self.assertEqual(repr(self.log.buffer), "") + + def test_log_config(self): + self.assertIs(self.log.writable(), True) + self.assertIs(self.log.readable(), False) + + self.assertEqual("UTF-8", self.log.encoding) + self.assertEqual("replace", self.log.errors) + + self.assertIs(self.log.line_buffering, True) + self.assertIs(self.log.write_through, False) + + def test_empty_str(self): + self.log.write("") + self.log.flush() + + self.assert_writes([]) + + def test_simple_str(self): + self.log.write("hello world\n") + + self.assert_writes([b"hello world\n"]) + + def test_buffered_str(self): + self.log.write("h") + self.log.write("ello") + self.log.write(" ") + self.log.write("world\n") + self.log.write("goodbye.") + self.log.flush() + + self.assert_writes([b"hello world\n", b"goodbye."]) + + def test_manual_flush(self): + self.log.write("Hello") + + self.assert_writes([]) + + self.log.write(" world\nHere for a while...\nGoodbye") + self.assert_writes([b"Hello world\n", b"Here for a while...\n"]) + + self.log.write(" world\nHello again") + self.assert_writes([b"Goodbye world\n"]) + + self.log.flush() + self.assert_writes([b"Hello again"]) + + def test_non_ascii(self): + # Spanish + self.log.write("ol\u00e9\n") + self.assert_writes([b"ol\xc3\xa9\n"]) + + # Chinese + self.log.write("\u4e2d\u6587\n") + self.assert_writes([b"\xe4\xb8\xad\xe6\x96\x87\n"]) + + # Printing Non-BMP emoji + self.log.write("\U0001f600\n") + self.assert_writes([b"\xf0\x9f\x98\x80\n"]) + + # Non-encodable surrogates are replaced + self.log.write("\ud800\udc00\n") + self.assert_writes([b"??\n"]) + + def test_modified_null(self): + # Null characters are logged using "modified UTF-8". + self.log.write("\u0000\n") + self.assert_writes([b"\xc0\x80\n"]) + self.log.write("a\u0000\n") + self.assert_writes([b"a\xc0\x80\n"]) + self.log.write("\u0000b\n") + self.assert_writes([b"\xc0\x80b\n"]) + self.log.write("a\u0000b\n") + self.assert_writes([b"a\xc0\x80b\n"]) + + def test_nonstandard_str(self): + # String subclasses are accepted, but they should be converted + # to a standard str without calling any of their methods. + class CustomStr(str): + def splitlines(self, *args, **kwargs): + raise AssertionError() + + def __len__(self): + raise AssertionError() + + def __str__(self): + raise AssertionError() + + self.log.write(CustomStr("custom\n")) + self.assert_writes([b"custom\n"]) + + def test_non_str(self): + # Non-string classes are not accepted. + for obj in [b"", b"hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be str, not " + fr"{type(obj).__name__}" + ): + self.log.write(obj) + + def test_byteslike_in_buffer(self): + # The underlying buffer *can* accept bytes-like objects + self.log.buffer.write(bytearray(b"hello")) + self.log.flush() + + self.log.buffer.write(b"") + self.log.flush() + + self.log.buffer.write(b"goodbye") + self.log.flush() + + self.assert_writes([b"hello", b"goodbye"]) + + def test_non_byteslike_in_buffer(self): + for obj in ["hello", None, 42]: + with self.subTest(obj=obj): + with self.assertRaisesRegex( + TypeError, + fr"write\(\) argument must be bytes-like, not " + fr"{type(obj).__name__}" + ): + self.log.buffer.write(obj) diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 67ab8cf6baf657..c268a1f00f938e 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -18,7 +18,7 @@ _testinternalcapi = None from test import support -from test.support import os_helper, script_helper +from test.support import os_helper, script_helper, skip_emscripten_stack_overflow from test.support.ast_helper import ASTTestMixin from test.test_ast.utils import to_tuple from test.test_ast.snippets import ( @@ -745,6 +745,7 @@ def next(self): enum._test_simple_enum(_Precedence, ast._Precedence) @support.cpython_only + @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): fail_depth = support.exceeds_recursion_limit() crash_depth = 100_000 @@ -1661,6 +1662,7 @@ def test_level_as_none(self): exec(code, ns) self.assertIn('sleep', ns) + @skip_emscripten_stack_overflow() def test_recursion_direct(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) e.operand = e @@ -1668,6 +1670,7 @@ def test_recursion_direct(self): with support.infinite_recursion(): compile(ast.Expression(e), "", "eval") + @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) diff --git a/Lib/test/test_asyncio/test_eager_task_factory.py b/Lib/test/test_asyncio/test_eager_task_factory.py index 31d2a00dbb8c9c..0e2b189f761521 100644 --- a/Lib/test/test_asyncio/test_eager_task_factory.py +++ b/Lib/test/test_asyncio/test_eager_task_factory.py @@ -220,10 +220,14 @@ async def fail(): await asyncio.sleep(0) raise ValueError("no good") + async def blocked(): + fut = asyncio.Future() + await fut + async def run(): winner, index, excs = await asyncio.staggered.staggered_race( [ - lambda: asyncio.sleep(2, result="sleep2"), + lambda: blocked(), lambda: asyncio.sleep(1, result="sleep1"), lambda: fail() ], diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index aaeda33dd0c677..efca30f37414f9 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -805,6 +805,18 @@ def test_writelines_send_partial(self): self.assertTrue(self.sock.send.called) self.assertTrue(self.loop.writers) + def test_writelines_pauses_protocol(self): + data = memoryview(b'data') + self.sock.send.return_value = 2 + self.sock.send.fileno.return_value = 7 + + transport = self.socket_transport() + transport._high_water = 1 + transport.writelines([data]) + self.assertTrue(self.protocol.pause_writing.called) + self.assertTrue(self.sock.send.called) + self.assertTrue(self.loop.writers) + @unittest.skipUnless(selector_events._HAS_SENDMSG, 'no sendmsg') def test_write_sendmsg_full(self): data = memoryview(b'data') diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index e51711d9b4f1a4..06df217881a52f 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -2691,7 +2691,7 @@ def __del__(self): class ImmortalTests(unittest.TestCase): if sys.maxsize < (1 << 32): - IMMORTAL_REFCOUNT = 3 << 29 + IMMORTAL_REFCOUNT = 7 << 28 else: IMMORTAL_REFCOUNT = 3 << 30 diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 9e1985bb3a7639..7bb1ab38aa4fdf 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -459,6 +459,16 @@ def test_fromhex(self): self.assertRaises(ValueError, self.type2test.fromhex, '\x00') self.assertRaises(ValueError, self.type2test.fromhex, '12 \x00 34') + # For odd number of character(s) + for value in ("a", "aaa", "deadbee"): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn("fromhex() arg must contain an even number of hexadecimal digits", str(cm.exception)) + for value, position in (("a ", 1), (" aa a ", 5), (" aa a a ", 5)): + with self.assertRaises(ValueError) as cm: + self.type2test.fromhex(value) + self.assertIn(f"non-hexadecimal number found in fromhex() arg at position {position}", str(cm.exception)) + for data, pos in ( # invalid first hexadecimal character ('12 x4 56', 3), diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 9d5256b566b8af..78a706436aea0e 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,6 +1,6 @@ import unittest from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, - set_recursion_limit, skip_on_s390x) + set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow) try: import _testcapi except ImportError: @@ -1038,6 +1038,7 @@ class TestRecursion(unittest.TestCase): @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") @unittest.skipIf(_testcapi is None, "requires _testcapi") + @skip_emscripten_stack_overflow() def test_super_deep(self): def recurse(n): diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index 77730ad2f32085..a3179efe4a8235 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -110,6 +110,10 @@ def test_config_get(self): options.extend(( ("_pystats", bool, None), )) + if support.is_apple: + options.extend(( + ("use_system_logger", bool, None), + )) for name, option_type, sys_attr in options: with self.subTest(name=name, option_type=option_type, diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index b22ddd8ad858d4..666e2f2ab09548 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -415,6 +415,156 @@ def test_err_formatunraisable(self): # CRASHES formatunraisable(NULL, NULL) +class TestUnicodeTranslateError(UnicodeTranslateError): + # UnicodeTranslateError takes 4 arguments instead of 5, + # so we just make a UnicodeTranslateError class that is + # compatible with the UnicodeError.__init__. + def __init__(self, encoding, *args, **kwargs): + super().__init__(*args, **kwargs) + + +class TestUnicodeError(unittest.TestCase): + + def _check_no_crash(self, exc): + # ensure that the __str__() method does not crash + _ = str(exc) + + def test_unicode_encode_error_get_start(self): + get_start = _testcapi.unicode_encode_get_start + self._test_unicode_error_get_start('x', UnicodeEncodeError, get_start) + + def test_unicode_decode_error_get_start(self): + get_start = _testcapi.unicode_decode_get_start + self._test_unicode_error_get_start(b'x', UnicodeDecodeError, get_start) + + def test_unicode_translate_error_get_start(self): + get_start = _testcapi.unicode_translate_get_start + self._test_unicode_error_get_start('x', TestUnicodeTranslateError, get_start) + + def _test_unicode_error_get_start(self, literal, exc_type, get_start): + for obj_len, start, c_start in [ + # normal cases + (5, 0, 0), + (5, 1, 1), + (5, 2, 2), + # out of range start is clamped to max(0, obj_len - 1) + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (5, 5, 4), + (5, 10, 4), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 0), + (2, -1, 0), + (2, -2, 0), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, start=start): + exc = exc_type('utf-8', obj, start, obj_len, 'reason') + self.assertEqual(get_start(exc), c_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_start(self): + set_start = _testcapi.unicode_encode_set_start + self._test_unicode_error_set_start('x', UnicodeEncodeError, set_start) + + def test_unicode_decode_error_set_start(self): + set_start = _testcapi.unicode_decode_set_start + self._test_unicode_error_set_start(b'x', UnicodeDecodeError, set_start) + + def test_unicode_translate_error_set_start(self): + set_start = _testcapi.unicode_translate_set_start + self._test_unicode_error_set_start('x', TestUnicodeTranslateError, set_start) + + def _test_unicode_error_set_start(self, literal, exc_type, set_start): + obj_len = 5 + obj = literal * obj_len + for new_start in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_start(exc, new_start) + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_start=new_start): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.start = new_start + self.assertEqual(exc.start, new_start) + self._check_no_crash(exc) + + def test_unicode_encode_error_get_end(self): + get_end = _testcapi.unicode_encode_get_end + self._test_unicode_error_get_end('x', UnicodeEncodeError, get_end) + + def test_unicode_decode_error_get_end(self): + get_end = _testcapi.unicode_decode_get_end + self._test_unicode_error_get_end(b'x', UnicodeDecodeError, get_end) + + def test_unicode_translate_error_get_end(self): + get_end = _testcapi.unicode_translate_get_end + self._test_unicode_error_get_end('x', TestUnicodeTranslateError, get_end) + + def _test_unicode_error_get_end(self, literal, exc_type, get_end): + for obj_len, end, c_end in [ + # normal cases + (5, 0, 1), + (5, 1, 1), + (5, 2, 2), + # out-of-range clipped in [MIN(1, OBJLEN), MAX(MIN(1, OBJLEN), OBJLEN)] + (0, 0, 0), + (0, 1, 0), + (0, 10, 0), + (1, 1, 1), + (1, 2, 1), + (5, 5, 5), + (5, 5, 5), + (5, 10, 5), + # negative values are allowed but clipped in the getter + (0, -1, 0), + (1, -1, 1), + (2, -1, 1), + (2, -2, 1), + ]: + obj = literal * obj_len + with self.subTest(obj, exc_type=exc_type, end=end): + exc = exc_type('utf-8', obj, 0, end, 'reason') + self.assertEqual(get_end(exc), c_end) + self._check_no_crash(exc) + + def test_unicode_encode_error_set_end(self): + set_end = _testcapi.unicode_encode_set_end + self._test_unicode_error_set_end('x', UnicodeEncodeError, set_end) + + def test_unicode_decode_error_set_end(self): + set_end = _testcapi.unicode_decode_set_end + self._test_unicode_error_set_end(b'x', UnicodeDecodeError, set_end) + + def test_unicode_translate_error_set_end(self): + set_end = _testcapi.unicode_translate_set_end + self._test_unicode_error_set_end('x', TestUnicodeTranslateError, set_end) + + def _test_unicode_error_set_end(self, literal, exc_type, set_end): + obj_len = 5 + obj = literal * obj_len + for new_end in range(-2 * obj_len, 2 * obj_len): + with self.subTest('C-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the C API setter + set_end(exc, new_end) + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + with self.subTest('Py-API', obj=obj, exc_type=exc_type, new_end=new_end): + exc = exc_type('utf-8', obj, 0, obj_len, 'reason') + # arbitrary value is allowed in the attribute setter + exc.end = new_end + self.assertEqual(exc.end, new_end) + self._check_no_crash(exc) + + class Test_PyUnstable_Exc_PrepReraiseStar(ExceptionIsLikeMixin, unittest.TestCase): def setUp(self): diff --git a/Lib/test/test_capi/test_immortal.py b/Lib/test/test_capi/test_immortal.py index ef5d32b7f01935..3e36913ac301c3 100644 --- a/Lib/test/test_capi/test_immortal.py +++ b/Lib/test/test_capi/test_immortal.py @@ -2,6 +2,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') class TestCAPI(unittest.TestCase): @@ -11,6 +12,21 @@ def test_immortal_builtins(self): def test_immortal_small_ints(self): _testcapi.test_immortal_small_ints() +class TestInternalCAPI(unittest.TestCase): + + def test_immortal_builtins(self): + for obj in range(-5, 256): + self.assertTrue(_testinternalcapi.is_static_immortal(obj)) + self.assertTrue(_testinternalcapi.is_static_immortal(None)) + self.assertTrue(_testinternalcapi.is_static_immortal(False)) + self.assertTrue(_testinternalcapi.is_static_immortal(True)) + self.assertTrue(_testinternalcapi.is_static_immortal(...)) + self.assertTrue(_testinternalcapi.is_static_immortal(())) + for obj in range(300, 400): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + for obj in ([], {}, set()): + self.assertFalse(_testinternalcapi.is_static_immortal(obj)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 6ab7b685c2e18b..5035b2b4829bf6 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -68,8 +68,13 @@ def test_api_misuse(self): def check_malloc_without_gil(self, code): out = self.check(code) - expected = ('Fatal Python error: _PyMem_DebugMalloc: ' - 'Python memory allocator called without holding the GIL') + if not support.Py_GIL_DISABLED: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without holding the GIL') + else: + expected = ('Fatal Python error: _PyMem_DebugMalloc: ' + 'Python memory allocator called without an active thread state. ' + 'Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?') self.assertIn(expected, out) def test_pymem_malloc_without_gil(self): diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 80e705a37c4c5e..61512e610f46f2 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -100,11 +100,18 @@ def test_no_FatalError_infinite_loop(self): _rc, out, err = run_result self.assertEqual(out, b'') # This used to cause an infinite loop. - msg = ("Fatal Python error: PyThreadState_Get: " - "the function must be called with the GIL held, " - "after Python initialization and before Python finalization, " - "but the GIL is released " - "(the current Python thread state is NULL)").encode() + if not support.Py_GIL_DISABLED: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with the GIL held, " + "after Python initialization and before Python finalization, " + "but the GIL is released " + "(the current Python thread state is NULL)").encode() + else: + msg = ("Fatal Python error: PyThreadState_Get: " + "the function must be called with an active thread state, " + "after Python initialization and before Python finalization, " + "but it was called without an active thread state. " + "Are you trying to call the C API inside of a Py_BEGIN_ALLOW_THREADS block?").encode() self.assertTrue(err.rstrip().startswith(msg), err) @@ -2137,6 +2144,7 @@ def test_py_config_isoloated_per_interpreter(self): # test fails, assume that the environment in this process may # be altered and suspect. + @requires_subinterpreters @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_configured_settings(self): """ diff --git a/Lib/test/test_capi/test_structmembers.py b/Lib/test/test_capi/test_structmembers.py index 6b27dc512a7d15..f14ad9a9a5f512 100644 --- a/Lib/test/test_capi/test_structmembers.py +++ b/Lib/test/test_capi/test_structmembers.py @@ -37,6 +37,9 @@ def _make_test_object(cls): 9.99999,# T_FLOAT 10.1010101010, # T_DOUBLE "hi", # T_STRING_INPLACE + 12, # T_LONGLONG + 13, # T_ULONGLONG + b"c", # T_CHAR ) @@ -162,6 +165,16 @@ def test_inplace_string(self): self.assertRaises(TypeError, setattr, ts, "T_STRING_INPLACE", "s") self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + def test_char(self): + ts = self.ts + self.assertEqual(ts.T_CHAR, "c") + ts.T_CHAR = "z" + self.assertEqual(ts.T_CHAR, "z") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", "") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", b"a") + self.assertRaises(TypeError, setattr, ts, "T_CHAR", bytearray(b"b")) + self.assertRaises(TypeError, delattr, ts, "T_STRING_INPLACE") + class ReadWriteTests_OldAPI(ReadWriteTests, unittest.TestCase): cls = _test_structmembersType_OldAPI diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index e6b49caeb51f32..6349467c5d6b70 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -1,5 +1,6 @@ import unittest import sys +import gc from collections import namedtuple from test.support import import_helper @@ -257,5 +258,29 @@ def test__tuple_resize(self): self.assertRaises(SystemError, resize, [1, 2, 3], 0, False) self.assertRaises(SystemError, resize, NULL, 0, False) + def test_bug_59313(self): + # Before 3.14, the C-API function PySequence_Tuple + # would create incomplete tuples which were visible to + # the cycle GC, and this test would crash the interpeter. + TAG = object() + tuples = [] + + def referrer_tuples(): + return [x for x in gc.get_referrers(TAG) + if isinstance(x, tuple)] + + def my_iter(): + nonlocal tuples + yield TAG # 'tag' gets stored in the result tuple + tuples += referrer_tuples() + for x in range(10): + tuples += referrer_tuples() + # Prior to 3.13 would raise a SystemError when the tuple needs to be resized + yield x + + self.assertEqual(tuple(my_iter()), (TAG, *range(10))) + self.assertEqual(tuples, []) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 7720cf157fa9ae..e20e59944e9ce9 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -1,7 +1,7 @@ "Test the functionality of Python classes implementing operators." import unittest -from test.support import cpython_only, import_helper, script_helper +from test.support import cpython_only, import_helper, script_helper, skip_emscripten_stack_overflow testmeths = [ @@ -554,6 +554,7 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) + @skip_emscripten_stack_overflow() def testSFBug532646(self): # Test for SF bug 532646 diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index f7ea923ef17672..b5cf2ad18fe60b 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -121,6 +121,7 @@ def __getitem__(self, key): self.assertEqual(d['z'], 12) @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_extended_arg(self): repeat = int(get_c_recursion_limit() * 0.9) longexpr = 'x = x or ' + '-x' * repeat @@ -709,6 +710,7 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Expected limit is Py_C_RECURSION_LIMIT limit = get_c_recursion_limit() diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 179556f57e884f..fd002fb00ac338 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -299,6 +299,22 @@ def test_mul(self): self.assertRaises(TypeError, operator.mul, 1j, None) self.assertRaises(TypeError, operator.mul, None, 1j) + for z, w, r in [(1e300+1j, complex(INF, INF), complex(NAN, INF)), + (1e300+1j, complex(NAN, INF), complex(-INF, INF)), + (1e300+1j, complex(INF, NAN), complex(INF, INF)), + (complex(INF, 1), complex(NAN, INF), complex(NAN, INF)), + (complex(INF, 1), complex(INF, NAN), complex(INF, NAN)), + (complex(NAN, 1), complex(1, INF), complex(-INF, NAN)), + (complex(1, NAN), complex(1, INF), complex(NAN, INF)), + (complex(1e200, NAN), complex(1e200, NAN), complex(INF, NAN)), + (complex(1e200, NAN), complex(NAN, 1e200), complex(NAN, INF)), + (complex(NAN, 1e200), complex(1e200, NAN), complex(NAN, INF)), + (complex(NAN, 1e200), complex(NAN, 1e200), complex(-INF, NAN)), + (complex(NAN, NAN), complex(NAN, NAN), complex(NAN, NAN))]: + with self.subTest(z=z, w=w, r=r): + self.assertComplexesAreIdentical(z * w, r) + self.assertComplexesAreIdentical(w * z, r) + def test_mod(self): # % is no longer supported on complex numbers with self.assertRaises(TypeError): @@ -340,6 +356,7 @@ def test_pow(self): self.assertAlmostEqual(pow(1j, 200), 1) self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j) self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j) + self.assertRaises(OverflowError, pow, 1e200+1j, 5) self.assertRaises(TypeError, pow, 1j, None) self.assertRaises(TypeError, pow, None, 1j) self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j) diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 3dec64cc9a2414..d76341417e9bef 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -371,6 +371,7 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -398,6 +399,7 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -415,6 +417,7 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + @support.skip_emscripten_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index b5e165f3bae929..1b3e64efd410b8 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -1,4 +1,5 @@ import unittest +import sys from ctypes import Structure, Union, sizeof, c_char, c_int from ._support import (CField, Py_TPFLAGS_DISALLOW_INSTANTIATION, Py_TPFLAGS_IMMUTABLETYPE) @@ -75,6 +76,28 @@ def __init_subclass__(cls, **kwargs): 'ctypes state is not initialized'): class Subclass(BrokenStructure): ... + def test_max_field_size_gh126937(self): + # Classes for big structs should be created successfully. + # (But they most likely can't be instantiated.) + # Here we test the exact limit: the number of *bits* must fit + # in Py_ssize_t. + + class X(self.cls): + _fields_ = [('char', c_char),] + max_field_size = sys.maxsize // 8 + + class Y(self.cls): + _fields_ = [('largeField', X * max_field_size)] + class Z(self.cls): + _fields_ = [('largeField', c_char * max_field_size)] + + with self.assertRaises(ValueError): + class TooBig(self.cls): + _fields_ = [('largeField', X * (max_field_size + 1))] + with self.assertRaises(ValueError): + class TooBig(self.cls): + _fields_ = [('largeField', c_char * (max_field_size + 1))] + # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. def test___set__(self): diff --git a/Lib/test/test_ctypes/test_win32_com_foreign_func.py b/Lib/test/test_ctypes/test_win32_com_foreign_func.py index 8d217fc17efa02..7e54f8f6c31d33 100644 --- a/Lib/test/test_ctypes/test_win32_com_foreign_func.py +++ b/Lib/test/test_ctypes/test_win32_com_foreign_func.py @@ -9,8 +9,7 @@ raise unittest.SkipTest("Windows-specific test") -from _ctypes import COMError, CopyComPointer -from ctypes import HRESULT +from ctypes import COMError, CopyComPointer, HRESULT COINIT_APARTMENTTHREADED = 0x2 diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index aa801b9c4f7ad9..168b78a477ee9c 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -3663,6 +3663,7 @@ def f(a): return a encoding='latin1', errors='replace') self.assertEqual(ba, b'abc\xbd?') + @support.skip_emscripten_stack_overflow() def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): @@ -3942,6 +3943,7 @@ def __del__(self): # it as a leak. del C.__del__ + @unittest.skipIf(support.is_emscripten, "Seems to works in Pyodide?") def test_slots_trash(self): # Testing slot trash... # Deallocating deeply nested slotted trash caused stack overflows @@ -4864,6 +4866,7 @@ class Thing: # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) + @support.skip_emscripten_stack_overflow() def test_repr_as_str(self): # Issue #11603: crash or infinite loop when rebinding __str__ as # __repr__. diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index c94dc2df4f0a7f..86b2f22dee5347 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -594,6 +594,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @support.skip_emscripten_stack_overflow() def test_repr_deep(self): d = {} for i in range(get_c_recursion_limit() + 1): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index d9881611c19c43..d6bf00eeeb0013 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,7 @@ import copy import pickle import unittest -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class DictSetTest(unittest.TestCase): @@ -277,6 +277,7 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) + @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): d = {} for i in range(get_c_recursion_limit()//2 + 100): diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index f26411ace8fa73..c719f571152d61 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -5,15 +5,19 @@ import dis import functools import io +import itertools +import opcode import re import sys +import tempfile +import textwrap import types import unittest from test.support import (captured_stdout, requires_debug_ranges, - requires_specialization, cpython_only) + requires_specialization, cpython_only, + os_helper) from test.support.bytecode_helper import BytecodeTestCase -import opcode CACHE = dis.opmap["CACHE"] @@ -1260,27 +1264,6 @@ def test_super_instructions(self): got = self.get_disassembly(load_test, adaptive=True) self.do_disassembly_compare(got, dis_load_test_quickened_code) - @cpython_only - @requires_specialization - def test_binary_subscr_specialize(self): - binary_subscr_quicken = """\ - 0 RESUME_CHECK 0 - - 1 LOAD_NAME 0 (a) - LOAD_SMALL_INT 0 - %s - RETURN_VALUE -""" - co_list = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_list, {}, {'a': [0]})) - got = self.get_disassembly(co_list, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_LIST_INT") - - co_dict = compile('a[0]', "", "eval") - self.code_quicken(lambda: exec(co_dict, {}, {'a': {0: '1'}})) - got = self.get_disassembly(co_dict, adaptive=True) - self.do_disassembly_compare(got, binary_subscr_quicken % "BINARY_SUBSCR_DICT") - @cpython_only @requires_specialization def test_load_attr_specialize(self): @@ -2447,5 +2430,119 @@ def _unroll_caches_as_Instructions(instrs, show_caches=False): False, None, None, instr.positions) +class TestDisCLI(unittest.TestCase): + + def setUp(self): + self.filename = tempfile.mktemp() + self.addCleanup(os_helper.unlink, self.filename) + + @staticmethod + def text_normalize(string): + """Dedent *string* and strip it from its surrounding whitespaces. + + This method is used by the other utility functions so that any + string to write or to match against can be freely indented. + """ + return textwrap.dedent(string).strip() + + def set_source(self, content): + with open(self.filename, 'w') as fp: + fp.write(self.text_normalize(content)) + + def invoke_dis(self, *flags): + output = io.StringIO() + with contextlib.redirect_stdout(output): + dis.main(args=[*flags, self.filename]) + return self.text_normalize(output.getvalue()) + + def check_output(self, source, expect, *flags): + with self.subTest(source=source, flags=flags): + self.set_source(source) + res = self.invoke_dis(*flags) + expect = self.text_normalize(expect) + self.assertListEqual(res.splitlines(), expect.splitlines()) + + def test_invocation(self): + # test various combinations of parameters + base_flags = [ + ('-C', '--show-caches'), + ('-O', '--show-offsets'), + ('-P', '--show-positions'), + ('-S', '--specialized'), + ] + + self.set_source(''' + def f(): + print(x) + return None + ''') + + for r in range(1, len(base_flags) + 1): + for choices in itertools.combinations(base_flags, r=r): + for args in itertools.product(*choices): + with self.subTest(args=args[1:]): + _ = self.invoke_dis(*args) + + with self.assertRaises(SystemExit): + # suppress argparse error message + with contextlib.redirect_stderr(io.StringIO()): + _ = self.invoke_dis('--unknown') + + def test_show_cache(self): + # test 'python -m dis -C/--show-caches' + source = 'print()' + expect = ''' + 0 RESUME 0 + + 1 LOAD_NAME 0 (print) + PUSH_NULL + CALL 0 + CACHE 0 (counter: 0) + CACHE 0 (func_version: 0) + CACHE 0 + POP_TOP + LOAD_CONST 0 (None) + RETURN_VALUE + ''' + for flag in ['-C', '--show-caches']: + self.check_output(source, expect, flag) + + def test_show_offsets(self): + # test 'python -m dis -O/--show-offsets' + source = 'pass' + expect = ''' + 0 0 RESUME 0 + + 1 2 LOAD_CONST 0 (None) + 4 RETURN_VALUE + ''' + for flag in ['-O', '--show-offsets']: + self.check_output(source, expect, flag) + + def test_show_positions(self): + # test 'python -m dis -P/--show-positions' + source = 'pass' + expect = ''' + 0:0-1:0 RESUME 0 + + 1:0-1:4 LOAD_CONST 0 (None) + 1:0-1:4 RETURN_VALUE + ''' + for flag in ['-P', '--show-positions']: + self.check_output(source, expect, flag) + + def test_specialized_code(self): + # test 'python -m dis -S/--specialized' + source = 'pass' + expect = ''' + 0 RESUME 0 + + 1 LOAD_CONST_IMMORTAL 0 (None) + RETURN_VALUE + ''' + for flag in ['-S', '--specialized']: + self.check_output(source, expect, flag) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5c38b28322deb4..7110fb889f3c8e 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -649,6 +649,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_COMPAT.update({ 'legacy_windows_stdio': False, }) + if support.is_apple: + CONFIG_COMPAT['use_system_logger'] = False CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index b4fc290b1f32b6..53212529c27e28 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,7 +1,7 @@ import collections.abc import types import unittest -from test.support import get_c_recursion_limit +from test.support import get_c_recursion_limit, skip_emscripten_stack_overflow class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): @@ -464,11 +464,13 @@ def make_deep_eg(self): e = ExceptionGroup('eg', [e]) return e + @skip_emscripten_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) + @skip_emscripten_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 5beeac3adfc065..6ccfa9575f8569 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -2274,6 +2274,21 @@ def test_range_of_offsets(self): self.assertIn(expected, err.getvalue()) the_exception = exc + def test_subclass(self): + class MySyntaxError(SyntaxError): + pass + + try: + raise MySyntaxError("bad bad", ("bad.py", 1, 2, "abcdefg", 1, 7)) + except SyntaxError as exc: + with support.captured_stderr() as err: + sys.__excepthook__(*sys.exc_info()) + self.assertIn(""" + File "bad.py", line 1 + abcdefg + ^^^^^ +""", err.getvalue()) + def test_encodings(self): self.addCleanup(unlink, TESTFN) source = ( diff --git a/Lib/test/test_free_threading/test_methodcaller.py b/Lib/test/test_free_threading/test_methodcaller.py new file mode 100644 index 00000000000000..8846b0010012f2 --- /dev/null +++ b/Lib/test/test_free_threading/test_methodcaller.py @@ -0,0 +1,33 @@ +import unittest +from threading import Thread +from test.support import threading_helper +from operator import methodcaller + + +class TestMethodcaller(unittest.TestCase): + def test_methodcaller_threading(self): + number_of_threads = 10 + size = 4_000 + + mc = methodcaller("append", 2) + + def work(mc, l, ii): + for _ in range(ii): + mc(l) + + worker_threads = [] + lists = [] + for ii in range(number_of_threads): + l = [] + lists.append(l) + worker_threads.append(Thread(target=work, args=[mc, l, size])) + for t in worker_threads: + t.start() + for t in worker_threads: + t.join() + for l in lists: + assert len(l) == size + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_races.py b/Lib/test/test_free_threading/test_races.py new file mode 100644 index 00000000000000..09e1d52e3509f9 --- /dev/null +++ b/Lib/test/test_free_threading/test_races.py @@ -0,0 +1,134 @@ +# It's most useful to run these tests with ThreadSanitizer enabled. +import sys +import functools +import threading +import time +import unittest + +from test.support import threading_helper + + +class TestBase(unittest.TestCase): + pass + + +def do_race(func1, func2): + """Run func1() and func2() repeatedly in separate threads.""" + n = 1000 + + barrier = threading.Barrier(2) + + def repeat(func): + barrier.wait() + for _i in range(n): + func() + + threads = [ + threading.Thread(target=functools.partial(repeat, func1)), + threading.Thread(target=functools.partial(repeat, func2)), + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +@threading_helper.requires_working_threading() +class TestRaces(TestBase): + def test_racing_cell_set(self): + """Test cell object gettr/settr properties.""" + + def nested_func(): + x = 0 + + def inner(): + nonlocal x + x += 1 + + # This doesn't race because LOAD_DEREF and STORE_DEREF on the + # cell object use critical sections. + do_race(nested_func, nested_func) + + def nested_func2(): + x = 0 + + def inner(): + y = x + frame = sys._getframe(1) + frame.f_locals["x"] = 2 + + return inner + + def mutate_func2(): + inner = nested_func2() + cell = inner.__closure__[0] + old_value = cell.cell_contents + cell.cell_contents = 1000 + time.sleep(0) + cell.cell_contents = old_value + time.sleep(0) + + # This revealed a race with cell_set_contents() since it was missing + # the critical section. + do_race(nested_func2, mutate_func2) + + def test_racing_cell_cmp_repr(self): + """Test cell object compare and repr methods.""" + + def nested_func(): + x = 0 + y = 0 + + def inner(): + return x + y + + return inner.__closure__ + + cell_a, cell_b = nested_func() + + def mutate(): + cell_a.cell_contents += 1 + + def access(): + cell_a == cell_b + s = repr(cell_a) + + # cell_richcompare() and cell_repr used to have data races + do_race(mutate, access) + + def test_racing_load_super_attr(self): + """Test (un)specialization of LOAD_SUPER_ATTR opcode.""" + + class C: + def __init__(self): + try: + super().__init__ + super().__init__() + except RuntimeError: + pass # happens if __class__ is replaced with non-type + + def access(): + C() + + def mutate(): + # Swap out the super() global with a different one + real_super = super + globals()["super"] = lambda s=1: s + time.sleep(0) + globals()["super"] = real_super + time.sleep(0) + # Swap out the __class__ closure value with a non-type + cell = C.__init__.__closure__[0] + real_class = cell.cell_contents + cell.cell_contents = 99 + time.sleep(0) + cell.cell_contents = real_class + + # The initial PR adding specialized opcodes for LOAD_SUPER_ATTR + # had some races (one with the super() global changing and one + # with the cell binding being changed). + do_race(access, mutate) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index 758f74f54d0b56..a3b9f4b0175ae7 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -1,3 +1,4 @@ +import _testcapi import threading from test.support import threading_helper from unittest import TestCase @@ -41,3 +42,246 @@ def reader(): assert 0 <= eggs <= iters run_in_threads([writer, reader, reader, reader]) + + def test_T_BOOL(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + # different code paths for True and False + spam_old.T_BOOL = True + spam_new.T_BOOL = True + spam_old.T_BOOL = False + spam_new.T_BOOL = False + + def reader(): + for _ in range(1_000): + spam_old.T_BOOL + spam_new.T_BOOL + + run_in_threads([writer, reader]) + + def test_T_BYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_BYTE = 0 + spam_new.T_BYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_BYTE + spam_new.T_BYTE + + run_in_threads([writer, reader]) + + def test_T_UBYTE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UBYTE = 0 + spam_new.T_UBYTE = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UBYTE + spam_new.T_UBYTE + + run_in_threads([writer, reader]) + + def test_T_SHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_SHORT = 0 + spam_new.T_SHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_SHORT + spam_new.T_SHORT + + run_in_threads([writer, reader]) + + def test_T_USHORT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_USHORT = 0 + spam_new.T_USHORT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_USHORT + spam_new.T_USHORT + + run_in_threads([writer, reader]) + + def test_T_INT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_INT = 0 + spam_new.T_INT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_INT + spam_new.T_INT + + run_in_threads([writer, reader]) + + def test_T_UINT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_UINT = 0 + spam_new.T_UINT = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_UINT + spam_new.T_UINT + + run_in_threads([writer, reader]) + + def test_T_LONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONG = 0 + spam_new.T_LONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONG + spam_new.T_LONG + + run_in_threads([writer, reader]) + + def test_T_ULONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONG = 0 + spam_new.T_ULONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONG + spam_new.T_ULONG + + run_in_threads([writer, reader]) + + def test_T_PYSSIZET(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_PYSSIZET = 0 + spam_new.T_PYSSIZET = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_PYSSIZET + spam_new.T_PYSSIZET + + run_in_threads([writer, reader]) + + def test_T_FLOAT(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_FLOAT = 0.0 + spam_new.T_FLOAT = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_FLOAT + spam_new.T_FLOAT + + run_in_threads([writer, reader]) + + def test_T_DOUBLE(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_DOUBLE = 0.0 + spam_new.T_DOUBLE = 0.0 + + def reader(): + for _ in range(1_000): + spam_old.T_DOUBLE + spam_new.T_DOUBLE + + run_in_threads([writer, reader]) + + def test_T_LONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_LONGLONG = 0 + spam_new.T_LONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_LONGLONG + spam_new.T_LONGLONG + + run_in_threads([writer, reader]) + + def test_T_ULONGLONG(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_ULONGLONG = 0 + spam_new.T_ULONGLONG = 0 + + def reader(): + for _ in range(1_000): + spam_old.T_ULONGLONG + spam_new.T_ULONGLONG + + run_in_threads([writer, reader]) + + def test_T_CHAR(self): + spam_old = _testcapi._test_structmembersType_OldAPI() + spam_new = _testcapi._test_structmembersType_NewAPI() + + def writer(): + for _ in range(1_000): + spam_old.T_CHAR = "c" + spam_new.T_CHAR = "c" + + def reader(): + for _ in range(1_000): + spam_old.T_CHAR + spam_new.T_CHAR + + run_in_threads([writer, reader]) diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index 51463b6bb8c1b4..53f6d778bbecbc 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -124,6 +124,21 @@ def work(): for thread in threads: thread.join() + def test_object_class_change(self): + class Base: + def __init__(self): + self.attr = 123 + class ClassA(Base): + pass + class ClassB(Base): + pass + + obj = ClassA() + # keep reference to __dict__ + d = obj.__dict__ + obj.__class__ = ClassB + + def run_one(self, writer_func, reader_func): writer = Thread(target=writer_func) readers = [] diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 6d60f6941c4c5d..ffd2adb8665b45 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -404,6 +404,7 @@ def test_setstate_subclasses(self): self.assertEqual(r, ((1, 2), {})) self.assertIs(type(r[0]), tuple) + @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): f = self.partial(capture) @@ -2054,6 +2055,7 @@ def orig(a, /, b, c=True): ... @support.skip_on_s390x @unittest.skipIf(support.is_wasi, "WASI has limited C stack") + @support.skip_emscripten_stack_overflow() def test_lru_recursion(self): @self.module.lru_cache diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 0372815b9bfd27..b5140057a69d36 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -31,6 +31,11 @@ def __new__(cls, *args, **kwargs): return C ContainerNoGC = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + ### Support code ############################################################################### @@ -1130,6 +1135,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") @requires_gil_enabled("Free threading does not support incremental GC") # Use small increments to emulate longer running process in a shorter time @gc_threshold(200, 10) @@ -1167,20 +1173,15 @@ def make_ll(depth): enabled = gc.isenabled() gc.enable() olds = [] + initial_heap_size = _testinternalcapi.get_tracked_heap_size() for i in range(20_000): newhead = make_ll(20) count += 20 newhead.surprise = head olds.append(newhead) if len(olds) == 20: - stats = gc.get_stats() - young = stats[0] - incremental = stats[1] - old = stats[2] - collected = young['collected'] + incremental['collected'] + old['collected'] - count += CORRECTION - live = count - collected - self.assertLess(live, 25000) + new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size + self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") del olds[:] if not enabled: gc.disable() @@ -1322,7 +1323,8 @@ def test_refcount_errors(self): from test.support import gc_collect, SuppressCrashReport a = [1, 2, 3] - b = [a] + b = [a, a] + a.append(b) # Avoid coredump when Py_FatalError() calls abort() SuppressCrashReport().__enter__() @@ -1332,6 +1334,8 @@ def test_refcount_errors(self): # (to avoid deallocating it): import ctypes ctypes.pythonapi.Py_DecRef(ctypes.py_object(a)) + del a + del b # The garbage collector should now have a fatal error # when it reaches the broken object @@ -1360,7 +1364,7 @@ def test_refcount_errors(self): self.assertRegex(stderr, br'object type name: list') self.assertRegex(stderr, - br'object repr : \[1, 2, 3\]') + br'object repr : \[1, 2, 3, \[\[...\], \[...\]\]\]') class GCTogglingTests(unittest.TestCase): diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 8879902a6e2f41..7b3dc0fdaedc3b 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -59,6 +59,52 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_obsolete_rfc850_date_format(self): + # Test cases with different days and dates in obsolete RFC 850 format + test_cases = [ + # from RFC 850, change EST to GMT + # https://datatracker.ietf.org/doc/html/rfc850#section-2 + { + 'data': 'key=value; expires=Saturday, 01-Jan-83 00:00:00 GMT', + 'output': 'Saturday, 01-Jan-83 00:00:00 GMT' + }, + { + 'data': 'key=value; expires=Friday, 19-Nov-82 16:59:30 GMT', + 'output': 'Friday, 19-Nov-82 16:59:30 GMT' + }, + # from RFC 9110 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.7-6 + { + 'data': 'key=value; expires=Sunday, 06-Nov-94 08:49:37 GMT', + 'output': 'Sunday, 06-Nov-94 08:49:37 GMT' + }, + # other test cases + { + 'data': 'key=value; expires=Wednesday, 09-Nov-94 08:49:37 GMT', + 'output': 'Wednesday, 09-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Friday, 11-Nov-94 08:49:37 GMT', + 'output': 'Friday, 11-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Monday, 14-Nov-94 08:49:37 GMT', + 'output': 'Monday, 14-Nov-94 08:49:37 GMT' + }, + ] + + for case in test_cases: + with self.subTest(data=case['data']): + C = cookies.SimpleCookie() + C.load(case['data']) + + # Extract the cookie name from the data string + cookie_name = case['data'].split('=')[0] + + # Check if the cookie is loaded correctly + self.assertIn(cookie_name, C) + self.assertEqual(C[cookie_name].get('expires'), case['output']) + def test_unquote(self): cases = [ (r'a="b=\""', 'b="'), diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 0e39998ebb3783..83efbc1e25e77a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -807,6 +807,50 @@ def test_issue105979(self): self.assertIn("Frozen object named 'x' is invalid", str(cm.exception)) + def test_frozen_module_from_import_error(self): + with self.assertRaises(ImportError) as cm: + from os import this_will_never_exist + self.assertIn( + f"cannot import name 'this_will_never_exist' from 'os' ({os.__file__})", + str(cm.exception), + ) + with self.assertRaises(ImportError) as cm: + from sys import this_will_never_exist + self.assertIn( + "cannot import name 'this_will_never_exist' from 'sys' (unknown location)", + str(cm.exception), + ) + + scripts = [ + """ +import os +os.__spec__.has_location = False +os.__file__ = [] +from os import this_will_never_exist +""", + """ +import os +os.__spec__.has_location = False +del os.__file__ +from os import this_will_never_exist +""", + """ +import os +os.__spec__.origin = [] +os.__file__ = [] +from os import this_will_never_exist +""" + ] + for script in scripts: + with self.subTest(script=script): + expected_error = ( + b"cannot import name 'this_will_never_exist' " + b"from 'os' (unknown location)" + ) + popen = script_helper.spawn_python("-c", script) + stdout, stderr = popen.communicate() + self.assertIn(expected_error, stdout) + def test_script_shadowing_stdlib(self): script_errors = [ ( @@ -1068,7 +1112,7 @@ class substr(str): except AttributeError as e: print(str(e)) -fractions.__spec__.origin = 0 +fractions.__spec__.origin = [] try: fractions.Fraction except AttributeError as e: @@ -1092,7 +1136,7 @@ class substr(str): except ImportError as e: print(str(e)) -fractions.__spec__.origin = 0 +fractions.__spec__.origin = [] try: from fractions import Fraction except ImportError as e: @@ -1133,6 +1177,19 @@ def test_script_shadowing_stdlib_sys_path_modification(self): stdout, stderr = popen.communicate() self.assertRegex(stdout, expected_error) + def test_create_dynamic_null(self): + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec: + name = "a\x00b" + origin = "abc" + _imp.create_dynamic(Spec()) + + with self.assertRaisesRegex(ValueError, 'embedded null character'): + class Spec2: + name = "abc" + origin = "a\x00b" + _imp.create_dynamic(Spec2()) + @skip_if_dont_write_bytecode class FilePermissionTests(unittest.TestCase): diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index a92627a4d60f68..1ecf18bf49fa7e 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -2992,6 +2992,17 @@ def test2(pod=42, /): with self.assertRaisesRegex(ValueError, 'follows default argument'): S((pkd, pk)) + second_args = args.replace(name="second_args") + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, second_args)) + + with self.assertRaisesRegex(ValueError, 'more than one variadic positional parameter'): + S((args, ko, second_args)) + + second_kwargs = kwargs.replace(name="second_kwargs") + with self.assertRaisesRegex(ValueError, 'more than one variadic keyword parameter'): + S((kwargs, second_kwargs)) + def test_signature_object_pickle(self): def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass foo_partial = functools.partial(foo, a=1) diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index a9befbba64daa0..01856d9bf67657 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1649,6 +1649,10 @@ def test_set___main___attrs(self): self.assertIs(after2, None) self.assertEqual(after3.type.__name__, 'AssertionError') + with self.assertRaises(ValueError): + # GH-127165: Embedded NULL characters broke the lookup + _interpreters.set___main___attrs(interpid, {"\x00": 1}) + with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: with self.assertRaisesRegex(InterpreterError, 'unrecognized'): diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index f1f8ce57668f3b..81c17b2731cc58 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -3932,6 +3932,22 @@ def test_issue35928(self): f.write(res) self.assertEqual(res + f.readline(), 'foo\nbar\n') + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_read_non_blocking(self): + import os + r, w = os.pipe() + try: + os.set_blocking(r, False) + with self.io.open(r, 'rt') as textfile: + r = None + # Nothing has been written so a non-blocking read raises a BlockingIOError exception. + with self.assertRaises(BlockingIOError): + textfile.read() + finally: + if r is not None: + os.close(r) + os.close(w) + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index 95a119ba683e09..abc75c82375d98 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -263,12 +263,14 @@ def test_subclass_tuple(self): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, str)))) + @support.skip_emscripten_stack_overflow() def test_subclass_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown with support.infinite_recursion(): self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @support.skip_emscripten_stack_overflow() def test_isinstance_recursion_limit(self): # make sure that issubclass raises RecursionError before the C stack is # blown @@ -315,6 +317,7 @@ def __bases__(self): self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) + @support.skip_emscripten_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index b94d688738f9e8..f0fd1d28f56f55 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -518,6 +518,15 @@ def test_count(self): self.assertEqual(next(c), -8) self.assertEqual(repr(count(10.25)), 'count(10.25)') self.assertEqual(repr(count(10.0)), 'count(10.0)') + + self.assertEqual(repr(count(maxsize)), f'count({maxsize})') + c = count(maxsize - 1) + self.assertEqual(repr(c), f'count({maxsize - 1})') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize})') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 1})') + self.assertEqual(type(next(count(10.0))), float) for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): # Test repr @@ -578,6 +587,20 @@ def test_count_with_step(self): self.assertEqual(type(next(c)), int) self.assertEqual(type(next(c)), float) + c = count(maxsize -2, 2) + self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, 2)') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 2}, 2)') + + c = count(maxsize + 1, -1) + self.assertEqual(repr(c), f'count({maxsize + 1}, -1)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, -1)') + next(c) + self.assertEqual(repr(c), f'count({maxsize - 1}, -1)') + @threading_helper.requires_working_threading() def test_count_threading(self, step=1): # this test verifies multithreading consistency, which is diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 290207e9c15b88..663c0643579ac8 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -68,6 +68,7 @@ def default(self, o): self.fail("didn't raise ValueError on default recursion") + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -81,6 +82,7 @@ def test_highly_nested_objects_decoding(self): with support.infinite_recursion(): self.loads('[' * 100000 + '1' + ']' * 100000) + @support.skip_emscripten_stack_overflow() def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} @@ -93,6 +95,7 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) + @support.skip_emscripten_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 93b8684c725d24..4ed9f1fc1b8020 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -1,5 +1,5 @@ from test import support -from test.support import is_apple_mobile, os_helper, requires_debug_ranges +from test.support import is_apple_mobile, os_helper, requires_debug_ranges, is_emscripten from test.support.script_helper import assert_python_ok import array import io @@ -294,7 +294,7 @@ def test_recursion_limit(self): #if os.name == 'nt' and support.Py_DEBUG: if os.name == 'nt': MAX_MARSHAL_STACK_DEPTH = 1000 - elif sys.platform == 'wasi' or is_apple_mobile: + elif sys.platform == 'wasi' or is_emscripten or is_apple_mobile: MAX_MARSHAL_STACK_DEPTH = 1500 else: MAX_MARSHAL_STACK_DEPTH = 2000 diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index fecafd53aa6e6f..6976a5d85da019 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2722,7 +2722,7 @@ def test_fma_infinities(self): # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. @unittest.skipIf( - sys.platform.startswith(("freebsd", "wasi", "netbsd")) + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) or (sys.platform == "android" and platform.machine() == "x86_64"), f"this platform doesn't implement IEE 754-2008 properly") def test_fma_zero_result(self): diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 2d4bf5f1408df8..96320ebef6467a 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -15,6 +15,7 @@ import pickle import struct +from itertools import product from test.support import import_helper @@ -58,12 +59,53 @@ def test_getitem(self): for tp in self._types: self.check_getitem_with_type(tp) + def test_index(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) # may be a sub-view + l = m.tolist() + k = 2 * len(self._source) + + for chi in self._source: + if chi in l: + self.assertEqual(m.index(chi), l.index(chi)) + else: + self.assertRaises(ValueError, m.index, chi) + + for start, stop in product(range(-k, k), range(-k, k)): + index = -1 + try: + index = l.index(chi, start, stop) + except ValueError: + pass + + if index == -1: + self.assertRaises(ValueError, m.index, chi, start, stop) + else: + self.assertEqual(m.index(chi, start, stop), index) + def test_iter(self): for tp in self._types: b = tp(self._source) m = self._view(b) self.assertEqual(list(m), [m[i] for i in range(len(m))]) + def test_count(self): + for tp in self._types: + b = tp(self._source) + m = self._view(b) + l = m.tolist() + for ch in list(m): + self.assertEqual(m.count(ch), l.count(ch)) + + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # may be sliced + l = m.tolist() + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), l.count(ord('a'))) + self.assertEqual(m.count(ord('b')), l.count(ord('b'))) + self.assertEqual(m.count(ord('c')), l.count(ord('c'))) + def test_setitem_readonly(self): if not self.ro_type: self.skipTest("no read-only type to test") @@ -438,6 +480,18 @@ def _view(self, obj): def _check_contents(self, tp, obj, contents): self.assertEqual(obj, tp(contents)) + def test_count(self): + super().test_count() + for tp in self._types: + b = tp((b'a' * 5) + (b'c' * 3)) + m = self._view(b) # should not be sliced + self.assertEqual(len(b), len(m)) + with self.subTest('count', buffer=b): + self.assertEqual(m.count(ord('a')), 5) + self.assertEqual(m.count(ord('b')), 0) + self.assertEqual(m.count(ord('c')), 3) + + class BaseMemorySliceTests: source_bytes = b"XabcdefY" diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index b640aa08e4a812..5a4bcebedf19de 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -11,7 +11,7 @@ import unittest import test.support -from test.support import requires_specialization, script_helper +from test.support import requires_specialization_ft, script_helper from test.support.import_helper import import_module _testcapi = test.support.import_helper.import_module("_testcapi") @@ -850,6 +850,13 @@ def __init__(self, events): def __call__(self, code, offset, val): self.events.append(("return", code.co_name, val)) +# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that +# are deferred. We only defer functions defined at the top-level. +class ValueErrorRaiser: + def __init__(self): + raise ValueError() + + class ExceptionMonitoringTest(CheckEvents): exception_recorders = ( @@ -1045,16 +1052,12 @@ def func(): ) self.assertEqual(events[0], ("throw", IndexError)) - @requires_specialization + @requires_specialization_ft def test_no_unwind_for_shim_frame(self): - class B: - def __init__(self): - raise ValueError() - def f(): try: - return B() + return ValueErrorRaiser() except ValueError: pass diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 1a6eac236009c3..50b5f365165921 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -493,6 +493,18 @@ def f(): self.assertFalse(f()) +# gh-127274: CALL_ALLOC_AND_ENTER_INIT will only cache __init__ methods that +# are deferred. We only defer functions defined at the top-level. +class MyClass: + def __init__(self): + pass + + +class InitTakesArg: + def __init__(self, arg): + self.arg = arg + + class TestCallCache(TestBase): def test_too_many_defaults_0(self): def f(): @@ -522,12 +534,8 @@ def f(x, y): f() @disabling_optimizer - @requires_specialization + @requires_specialization_ft def test_assign_init_code(self): - class MyClass: - def __init__(self): - pass - def instantiate(): return MyClass() @@ -544,6 +552,20 @@ def count_args(self, *args): MyClass.__init__.__code__ = count_args.__code__ instantiate() + @disabling_optimizer + @requires_specialization_ft + def test_push_init_frame_fails(self): + def instantiate(): + return InitTakesArg() + + for _ in range(2): + with self.assertRaises(TypeError): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + with self.assertRaises(TypeError): + instantiate() + @threading_helper.requires_working_threading() class TestRacesDoNotCrash(TestBase): @@ -617,7 +639,7 @@ def write(items): opname = "BINARY_SUBSCR_GETITEM" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_binary_subscr_list_int(self): def get_items(): items = [] @@ -1023,7 +1045,7 @@ def write(items): opname = "STORE_ATTR_WITH_HINT" self.assert_races_do_not_crash(opname, get_items, read, write) - @requires_specialization + @requires_specialization_ft def test_store_subscr_list_int(self): def get_items(): items = [] @@ -1229,48 +1251,129 @@ class TestSpecializer(TestBase): @cpython_only @requires_specialization_ft def test_binary_op(self): - def f(): + def binary_op_add_int(): for _ in range(100): a, b = 1, 2 c = a + b self.assertEqual(c, 3) - f() - self.assert_specialized(f, "BINARY_OP_ADD_INT") - self.assert_no_opcode(f, "BINARY_OP") + binary_op_add_int() + self.assert_specialized(binary_op_add_int, "BINARY_OP_ADD_INT") + self.assert_no_opcode(binary_op_add_int, "BINARY_OP") - def g(): + def binary_op_add_unicode(): for _ in range(100): a, b = "foo", "bar" c = a + b self.assertEqual(c, "foobar") - g() - self.assert_specialized(g, "BINARY_OP_ADD_UNICODE") - self.assert_no_opcode(g, "BINARY_OP") + binary_op_add_unicode() + self.assert_specialized(binary_op_add_unicode, "BINARY_OP_ADD_UNICODE") + self.assert_no_opcode(binary_op_add_unicode, "BINARY_OP") + + @cpython_only + @requires_specialization_ft + def test_load_super_attr(self): + """Ensure that LOAD_SUPER_ATTR is specialized as expected.""" + + class A: + def __init__(self): + meth = super().__init__ + super().__init__() + + for _ in range(100): + A() + + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_specialized(A.__init__, "LOAD_SUPER_ATTR_METHOD") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR") + + # Temporarily replace super() with something else. + real_super = super + + def fake_super(): + def init(self): + pass + + return init + + # Force unspecialize + globals()['super'] = fake_super + try: + # Should be unspecialized after enough calls. + for _ in range(100): + A() + finally: + globals()['super'] = real_super + + # Ensure the specialized instructions are not present + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_ATTR") + self.assert_no_opcode(A.__init__, "LOAD_SUPER_ATTR_METHOD") @cpython_only @requires_specialization_ft def test_contain_op(self): - def f(): + def contains_op_dict(): for _ in range(100): a, b = 1, {1: 2, 2: 5} self.assertTrue(a in b) self.assertFalse(3 in b) - f() - self.assert_specialized(f, "CONTAINS_OP_DICT") - self.assert_no_opcode(f, "CONTAINS_OP") + contains_op_dict() + self.assert_specialized(contains_op_dict, "CONTAINS_OP_DICT") + self.assert_no_opcode(contains_op_dict, "CONTAINS_OP") - def g(): + def contains_op_set(): for _ in range(100): a, b = 1, {1, 2} self.assertTrue(a in b) self.assertFalse(3 in b) - g() - self.assert_specialized(g, "CONTAINS_OP_SET") - self.assert_no_opcode(g, "CONTAINS_OP") + contains_op_set() + self.assert_specialized(contains_op_set, "CONTAINS_OP_SET") + self.assert_no_opcode(contains_op_set, "CONTAINS_OP") + + @cpython_only + @requires_specialization_ft + def test_send_with(self): + def run_async(coro): + while True: + try: + coro.send(None) + except StopIteration: + break + + class CM: + async def __aenter__(self): + return self + + async def __aexit__(self, *exc): + pass + + async def send_with(): + for i in range(100): + async with CM(): + x = 1 + + run_async(send_with()) + # Note there are still unspecialized "SEND" opcodes in the + # cleanup paths of the 'with' statement. + self.assert_specialized(send_with, "SEND_GEN") + + @cpython_only + @requires_specialization_ft + def test_send_yield_from(self): + def g(): + yield None + + def send_yield_from(): + yield from g() + + for i in range(100): + list(send_yield_from()) + + self.assert_specialized(send_yield_from, "SEND_GEN") + self.assert_no_opcode(send_yield_from, "SEND") @cpython_only @requires_specialization_ft @@ -1342,34 +1445,81 @@ def to_bool_str(): @cpython_only @requires_specialization_ft def test_unpack_sequence(self): - def f(): + def unpack_sequence_two_tuple(): for _ in range(100): a, b = 1, 2 self.assertEqual(a, 1) self.assertEqual(b, 2) - f() - self.assert_specialized(f, "UNPACK_SEQUENCE_TWO_TUPLE") - self.assert_no_opcode(f, "UNPACK_SEQUENCE") + unpack_sequence_two_tuple() + self.assert_specialized(unpack_sequence_two_tuple, + "UNPACK_SEQUENCE_TWO_TUPLE") + self.assert_no_opcode(unpack_sequence_two_tuple, "UNPACK_SEQUENCE") - def g(): + def unpack_sequence_tuple(): for _ in range(100): a, = 1, self.assertEqual(a, 1) - g() - self.assert_specialized(g, "UNPACK_SEQUENCE_TUPLE") - self.assert_no_opcode(g, "UNPACK_SEQUENCE") + unpack_sequence_tuple() + self.assert_specialized(unpack_sequence_tuple, "UNPACK_SEQUENCE_TUPLE") + self.assert_no_opcode(unpack_sequence_tuple, "UNPACK_SEQUENCE") - def x(): + def unpack_sequence_list(): for _ in range(100): a, b = [1, 2] self.assertEqual(a, 1) self.assertEqual(b, 2) - x() - self.assert_specialized(x, "UNPACK_SEQUENCE_LIST") - self.assert_no_opcode(x, "UNPACK_SEQUENCE") + unpack_sequence_list() + self.assert_specialized(unpack_sequence_list, "UNPACK_SEQUENCE_LIST") + self.assert_no_opcode(unpack_sequence_list, "UNPACK_SEQUENCE") + + @cpython_only + @requires_specialization_ft + def test_binary_subscr(self): + def binary_subscr_list_int(): + for _ in range(100): + a = [1, 2, 3] + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_list_int() + self.assert_specialized(binary_subscr_list_int, + "BINARY_SUBSCR_LIST_INT") + self.assert_no_opcode(binary_subscr_list_int, "BINARY_SUBSCR") + + def binary_subscr_tuple_int(): + for _ in range(100): + a = (1, 2, 3) + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_tuple_int() + self.assert_specialized(binary_subscr_tuple_int, + "BINARY_SUBSCR_TUPLE_INT") + self.assert_no_opcode(binary_subscr_tuple_int, "BINARY_SUBSCR") + + def binary_subscr_dict(): + for _ in range(100): + a = {1: 2, 2: 3} + self.assertEqual(a[1], 2) + self.assertEqual(a[2], 3) + + binary_subscr_dict() + self.assert_specialized(binary_subscr_dict, "BINARY_SUBSCR_DICT") + self.assert_no_opcode(binary_subscr_dict, "BINARY_SUBSCR") + + def binary_subscr_str_int(): + for _ in range(100): + a = "foobar" + for idx, expected in enumerate(a): + self.assertEqual(a[idx], expected) + + binary_subscr_str_int() + self.assert_specialized(binary_subscr_str_int, "BINARY_SUBSCR_STR_INT") + self.assert_no_opcode(binary_subscr_str_int, "BINARY_SUBSCR") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 812d46482e238a..82578a0ef1e6f2 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -482,6 +482,8 @@ def bar(self, f=42): return f def baz(*args, **kwds): return kwds['name'], kwds['self'] + def return_arguments(self, *args, **kwds): + return args, kwds a = A() f = operator.methodcaller('foo') self.assertRaises(IndexError, f, a) @@ -498,6 +500,17 @@ def baz(*args, **kwds): f = operator.methodcaller('baz', name='spam', self='eggs') self.assertEqual(f(a), ('spam', 'eggs')) + many_positional_arguments = tuple(range(10)) + many_kw_arguments = dict(zip('abcdefghij', range(10))) + f = operator.methodcaller('return_arguments', *many_positional_arguments) + self.assertEqual(f(a), (many_positional_arguments, {})) + + f = operator.methodcaller('return_arguments', **many_kw_arguments) + self.assertEqual(f(a), ((), many_kw_arguments)) + + f = operator.methodcaller('return_arguments', *many_positional_arguments, **many_kw_arguments) + self.assertEqual(f(a), (many_positional_arguments, many_kw_arguments)) + def test_inplace(self): operator = self.module class C(object): diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index f3d2ceb263f6f4..8aac92934f6ac0 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -805,14 +805,28 @@ def _test_utime(self, set_time, filename=None): set_time(filename, (atime_ns, mtime_ns)) st = os.stat(filename) - if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) - self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + if support.is_emscripten: + # Emscripten timestamps are roundtripped through a 53 bit integer of + # nanoseconds. If we want to represent ~50 years which is an 11 + # digits number of seconds: + # 2*log10(60) + log10(24) + log10(365) + log10(60) + log10(50) + # is about 11. Because 53 * log10(2) is about 16, we only have 5 + # digits worth of sub-second precision. + # Some day it would be good to fix this upstream. + delta=1e-5 + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) + self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) - self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) - self.assertEqual(st.st_mtime_ns, mtime_ns) + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): def set_time(filename, ns): diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 6a994f890da616..68bff2cf0d511e 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -1,3 +1,4 @@ +import collections import contextlib import io import os @@ -21,7 +22,7 @@ from test.support import os_helper from test.support.os_helper import TESTFN, FakePath from test.test_pathlib import test_pathlib_abc -from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows, needs_symlinks +from test.test_pathlib.test_pathlib_abc import needs_posix, needs_windows try: import fcntl @@ -55,6 +56,21 @@ def new_test(self): self.cls.replace = old_replace return new_test + +_tests_needing_symlinks = set() +def needs_symlinks(fn): + """Decorator that marks a test as requiring a path class that supports symlinks.""" + _tests_needing_symlinks.add(fn.__name__) + return fn + + + +class UnsupportedOperationTest(unittest.TestCase): + def test_is_notimplemented(self): + self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) + self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + + # # Tests for the pure classes. # @@ -533,6 +549,9 @@ class PathTest(test_pathlib_abc.DummyPathTest, PurePathTest): can_symlink = os_helper.can_symlink() def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') super().setUp() os.chmod(self.parser.join(self.base, 'dirE'), 0) @@ -546,12 +565,9 @@ def tempdir(self): self.addCleanup(os_helper.rmtree, d) return d - def test_matches_pathbase_api(self): - our_names = {name for name in dir(self.cls) if name[0] != '_'} - our_names.remove('is_reserved') # only present in PurePath + def test_matches_pathbase_docstrings(self): path_names = {name for name in dir(pathlib._abc.PathBase) if name[0] != '_'} - self.assertEqual(our_names, path_names) - for attr_name in our_names: + for attr_name in path_names: if attr_name == 'parser': # On Windows, Path.parser is ntpath, but PathBase.parser is # posixpath, and so their docstrings differ. @@ -696,6 +712,34 @@ def test_copy_file_preserve_metadata(self): if hasattr(source_st, 'st_flags'): self.assertEqual(source_st.st_flags, target_st.st_flags) + @needs_symlinks + def test_copy_file_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + + @needs_symlinks + def test_copy_file_to_existing_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'dirB' / 'fileB' + target = base / 'linkA' + real_target = base / 'fileA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertTrue(real_target.exists()) + self.assertFalse(real_target.is_symlink()) + self.assertEqual(source.read_text(), real_target.read_text()) + @os_helper.skip_unless_xattr def test_copy_file_preserve_metadata_xattrs(self): base = self.cls(self.base) @@ -705,6 +749,118 @@ def test_copy_file_preserve_metadata_xattrs(self): source.copy(target, preserve_metadata=True) self.assertEqual(os.getxattr(target, b'user.foo'), b'42') + @needs_symlinks + def test_copy_symlink_follow_symlinks_true(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertFalse(target.is_symlink()) + self.assertEqual(source.read_text(), target.read_text()) + + @needs_symlinks + def test_copy_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkA' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.copy, source) + + @needs_symlinks + def test_copy_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'fileA') + target.symlink_to(base / 'dirC') + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_follow_symlinks_false(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'copyA' + result = source.copy(target, follow_symlinks=False) + self.assertEqual(result, target) + self.assertTrue(target.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source.readlink(), target.readlink()) + + @needs_symlinks + def test_copy_directory_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.copy, source) + self.assertRaises(OSError, source.copy, source, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_into_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + target = base / 'linkB' / 'copyB' + self.assertRaises(OSError, source.copy, target) + self.assertRaises(OSError, source.copy, target, follow_symlinks=False) + self.assertFalse(target.exists()) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC') + target.symlink_to(base / 'fileA') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_directory_symlink_to_existing_directory_symlink(self): + base = self.cls(self.base) + source = base / 'copySource' + target = base / 'copyTarget' + source.symlink_to(base / 'dirC' / 'dirD') + target.symlink_to(base / 'dirC') + self.assertRaises(FileExistsError, source.copy, target) + self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) + + @needs_symlinks + def test_copy_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'source' + target = base / 'target' + + source.mkdir() + source.joinpath('link').symlink_to('nonexistent') + + self.assertRaises(FileNotFoundError, source.copy, target) + + target2 = base / 'target2' + result = source.copy(target2, follow_symlinks=False) + self.assertEqual(result, target2) + self.assertTrue(target2.joinpath('link').is_symlink()) + self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) + @needs_symlinks def test_copy_link_preserve_metadata(self): base = self.cls(self.base) @@ -804,6 +960,54 @@ def test_copy_dir_preserve_metadata_xattrs(self): target_file = target.joinpath('dirD', 'fileD') self.assertEqual(os.getxattr(target_file, b'user.foo'), b'42') + @needs_symlinks + def test_move_file_symlink(self): + base = self.cls(self.base) + source = base / 'linkA' + source_readlink = source.readlink() + target = base / 'linkA_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_file_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkA' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dir_symlink(self): + base = self.cls(self.base) + source = base / 'linkB' + source_readlink = source.readlink() + target = base / 'linkB_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + + @needs_symlinks + def test_move_dir_symlink_to_itself(self): + base = self.cls(self.base) + source = base / 'linkB' + self.assertRaises(OSError, source.move, source) + + @needs_symlinks + def test_move_dangling_symlink(self): + base = self.cls(self.base) + source = base / 'brokenLink' + source_readlink = source.readlink() + target = base / 'brokenLink_moved' + result = source.move(target) + self.assertEqual(result, target) + self.assertFalse(source.exists()) + self.assertTrue(target.is_symlink()) + self.assertEqual(source_readlink, target.readlink()) + @patch_replace def test_move_file_other_fs(self): self.test_move_file() @@ -861,9 +1065,41 @@ def test_move_into_other_os(self): def test_move_into_empty_name_other_os(self): self.test_move_into_empty_name() + @needs_symlinks + def test_complex_symlinks_absolute(self): + self._check_complex_symlinks(self.base) + + @needs_symlinks + def test_complex_symlinks_relative(self): + self._check_complex_symlinks('.') + + @needs_symlinks + def test_complex_symlinks_relative_dot_dot(self): + self._check_complex_symlinks(self.parser.join('dirA', '..')) + def _check_complex_symlinks(self, link0_target): - super()._check_complex_symlinks(link0_target) + # Test solving a non-looping chain of symlinks (issue #19887). + parser = self.parser P = self.cls(self.base) + P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) + P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) + P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) + P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) + + # Resolve absolute paths. + p = (P / 'link0').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link1').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link2').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + p = (P / 'link3').resolve() + self.assertEqual(p, P) + self.assertEqualNormCase(str(p), self.base) + # Resolve relative paths. old_path = os.getcwd() os.chdir(self.base) @@ -883,6 +1119,118 @@ def _check_complex_symlinks(self, link0_target): finally: os.chdir(old_path) + def _check_resolve(self, p, expected, strict=True): + q = p.resolve(strict) + self.assertEqual(q, expected) + + # This can be used to check both relative and absolute resolutions. + _check_resolve_relative = _check_resolve_absolute = _check_resolve + + @needs_symlinks + def test_resolve_common(self): + P = self.cls + p = P(self.base, 'foo') + with self.assertRaises(OSError) as cm: + p.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ENOENT) + # Non-strict + parser = self.parser + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo')) + p = P(self.base, 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(self.base, 'foo', 'in', 'spam')) + p = P(self.base, '..', 'foo', 'in', 'spam') + self.assertEqualNormCase(str(p.resolve(strict=False)), + parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) + # These are all relative symlinks. + p = P(self.base, 'dirB', 'fileB') + self._check_resolve_relative(p, p) + p = P(self.base, 'linkA') + self._check_resolve_relative(p, P(self.base, 'fileA')) + p = P(self.base, 'dirA', 'linkC', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + p = P(self.base, 'dirB', 'linkD', 'fileB') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', + 'spam'), False) + p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', + 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + # Now create absolute symlinks. + d = self.tempdir() + P(self.base, 'dirA', 'linkX').symlink_to(d) + P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) + p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') + self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) + # Non-strict + p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') + self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), + False) + p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') + if self.cls.parser is not posixpath: + # In Windows, if linkY points to dirB, 'dirA\linkY\..' + # resolves to 'dirA' without resolving linkY first. + self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) + else: + # In Posix, if linkY points to dirB, 'dirA/linkY/..' + # resolves to 'dirB/..' first before resolving to parent of dirB. + self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) + + @needs_symlinks + def test_resolve_dot(self): + # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ + parser = self.parser + p = self.cls(self.base) + p.joinpath('0').symlink_to('.', target_is_directory=True) + p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) + p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) + q = p / '2' + self.assertEqual(q.resolve(strict=True), p) + r = q / '3' / '4' + self.assertRaises(FileNotFoundError, r.resolve, strict=True) + # Non-strict + self.assertEqual(r.resolve(strict=False), p / '3' / '4') + + def _check_symlink_loop(self, *args): + path = self.cls(*args) + with self.assertRaises(OSError) as cm: + path.resolve(strict=True) + self.assertEqual(cm.exception.errno, errno.ELOOP) + + @needs_posix + @needs_symlinks + def test_resolve_loop(self): + # Loops with relative symlinks. + self.cls(self.base, 'linkX').symlink_to('linkX/inside') + self._check_symlink_loop(self.base, 'linkX') + self.cls(self.base, 'linkY').symlink_to('linkY') + self._check_symlink_loop(self.base, 'linkY') + self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') + self._check_symlink_loop(self.base, 'linkZ') + # Non-strict + p = self.cls(self.base, 'linkZ', 'foo') + self.assertEqual(p.resolve(strict=False), p) + # Loops with absolute symlinks. + self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) + self._check_symlink_loop(self.base, 'linkU') + self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) + self._check_symlink_loop(self.base, 'linkV') + self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) + self._check_symlink_loop(self.base, 'linkW') + # Non-strict + q = self.cls(self.base, 'linkW', 'foo') + self.assertEqual(q.resolve(strict=False), q) + def test_resolve_nonexist_relative_issue38671(self): p = self.cls('non', 'exist') @@ -893,6 +1241,24 @@ def test_resolve_nonexist_relative_issue38671(self): finally: os.chdir(old_cwd) + @needs_symlinks + def test_readlink(self): + P = self.cls(self.base) + self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) + self.assertEqual((P / 'brokenLink').readlink(), + self.cls('non-existing')) + self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) + self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) + with self.assertRaises(OSError): + (P / 'fileA').readlink() + + @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") + def test_readlink_unsupported(self): + P = self.cls(self.base) + p = P / 'fileA' + with self.assertRaises(pathlib.UnsupportedOperation): + q.readlink(p) + @os_helper.skip_unless_working_chmod def test_chmod(self): p = self.cls(self.base) / 'fileA' @@ -994,6 +1360,60 @@ def test_group_no_follow_symlinks(self): self.assertEqual(expected_gid, gid_2) self.assertEqual(expected_name, link.group(follow_symlinks=False)) + def test_unlink(self): + p = self.cls(self.base) / 'fileA' + p.unlink() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + def test_unlink_missing_ok(self): + p = self.cls(self.base) / 'fileAAA' + self.assertFileNotFound(p.unlink) + p.unlink(missing_ok=True) + + def test_rmdir(self): + p = self.cls(self.base) / 'dirA' + for q in p.iterdir(): + q.unlink() + p.rmdir() + self.assertFileNotFound(p.stat) + self.assertFileNotFound(p.unlink) + + @needs_symlinks + def test_delete_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir_ = tmp / 'dir' + dir_.mkdir() + link = tmp / 'link' + link.symlink_to(dir_) + link._delete() + self.assertTrue(dir_.exists()) + self.assertFalse(link.exists(follow_symlinks=False)) + + @needs_symlinks + def test_delete_inner_symlink(self): + tmp = self.cls(self.base, 'delete') + tmp.mkdir() + dir1 = tmp / 'dir1' + dir2 = dir1 / 'dir2' + dir3 = tmp / 'dir3' + for d in dir1, dir2, dir3: + d.mkdir() + file1 = tmp / 'file1' + file1.write_text('foo') + link1 = dir1 / 'link1' + link1.symlink_to(dir2) + link2 = dir1 / 'link2' + link2.symlink_to(dir3) + link3 = dir1 / 'link3' + link3.symlink_to(file1) + # make sure symlinks are removed but not followed + dir1._delete() + self.assertFalse(dir1.exists()) + self.assertTrue(dir3.exists()) + self.assertTrue(file1.exists()) + @unittest.skipIf(sys.platform[:6] == 'cygwin', "This test can't be run on Cygwin (issue #1071513).") @os_helper.skip_if_dac_override @@ -1357,13 +1777,48 @@ def test_symlink_to_unsupported(self): with self.assertRaises(pathlib.UnsupportedOperation): q.symlink_to(p) - def test_is_junction(self): + @needs_symlinks + def test_stat_no_follow_symlinks(self): + p = self.cls(self.base) / 'linkA' + st = p.stat() + self.assertNotEqual(st, p.stat(follow_symlinks=False)) + + @needs_symlinks + def test_lstat(self): + p = self.cls(self.base)/ 'linkA' + st = p.stat() + self.assertNotEqual(st, p.lstat()) + + def test_lstat_nosymlink(self): + p = self.cls(self.base) / 'fileA' + st = p.stat() + self.assertEqual(st, p.lstat()) + + def test_is_junction_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_junction()) + self.assertFalse((P / 'dirA').is_junction()) + self.assertFalse((P / 'non-existing').is_junction()) + self.assertFalse((P / 'fileA' / 'bah').is_junction()) + self.assertFalse((P / 'fileA\udfff').is_junction()) + self.assertFalse((P / 'fileA\x00').is_junction()) + + def test_is_junction_true(self): P = self.cls(self.base) with mock.patch.object(P.parser, 'isjunction'): self.assertEqual(P.is_junction(), P.parser.isjunction.return_value) P.parser.isjunction.assert_called_once_with(P) + def test_is_fifo_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_fifo()) + self.assertFalse((P / 'dirA').is_fifo()) + self.assertFalse((P / 'non-existing').is_fifo()) + self.assertFalse((P / 'fileA' / 'bah').is_fifo()) + self.assertIs((P / 'fileA\udfff').is_fifo(), False) + self.assertIs((P / 'fileA\x00').is_fifo(), False) + @unittest.skipUnless(hasattr(os, "mkfifo"), "os.mkfifo() required") @unittest.skipIf(sys.platform == "vxworks", "fifo requires special path on VxWorks") @@ -1379,6 +1834,15 @@ def test_is_fifo_true(self): self.assertIs(self.cls(self.base, 'myfifo\udfff').is_fifo(), False) self.assertIs(self.cls(self.base, 'myfifo\x00').is_fifo(), False) + def test_is_socket_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_socket()) + self.assertFalse((P / 'dirA').is_socket()) + self.assertFalse((P / 'non-existing').is_socket()) + self.assertFalse((P / 'fileA' / 'bah').is_socket()) + self.assertIs((P / 'fileA\udfff').is_socket(), False) + self.assertIs((P / 'fileA\x00').is_socket(), False) + @unittest.skipUnless(hasattr(socket, "AF_UNIX"), "Unix sockets required") @unittest.skipIf( is_emscripten, "Unix sockets are not implemented on Emscripten." @@ -1402,6 +1866,24 @@ def test_is_socket_true(self): self.assertIs(self.cls(self.base, 'mysock\udfff').is_socket(), False) self.assertIs(self.cls(self.base, 'mysock\x00').is_socket(), False) + def test_is_block_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_block_device()) + self.assertFalse((P / 'dirA').is_block_device()) + self.assertFalse((P / 'non-existing').is_block_device()) + self.assertFalse((P / 'fileA' / 'bah').is_block_device()) + self.assertIs((P / 'fileA\udfff').is_block_device(), False) + self.assertIs((P / 'fileA\x00').is_block_device(), False) + + def test_is_char_device_false(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_char_device()) + self.assertFalse((P / 'dirA').is_char_device()) + self.assertFalse((P / 'non-existing').is_char_device()) + self.assertFalse((P / 'fileA' / 'bah').is_char_device()) + self.assertIs((P / 'fileA\udfff').is_char_device(), False) + self.assertIs((P / 'fileA\x00').is_char_device(), False) + def test_is_char_device_true(self): # os.devnull should generally be a char device. P = self.cls(os.devnull) @@ -1413,7 +1895,14 @@ def test_is_char_device_true(self): self.assertIs(self.cls(f'{os.devnull}\udfff').is_char_device(), False) self.assertIs(self.cls(f'{os.devnull}\x00').is_char_device(), False) - def test_is_mount_root(self): + def test_is_mount(self): + P = self.cls(self.base) + self.assertFalse((P / 'fileA').is_mount()) + self.assertFalse((P / 'dirA').is_mount()) + self.assertFalse((P / 'non-existing').is_mount()) + self.assertFalse((P / 'fileA' / 'bah').is_mount()) + if self.can_symlink: + self.assertFalse((P / 'linkA').is_mount()) if os.name == 'nt': R = self.cls('c:\\') else: @@ -1421,10 +1910,40 @@ def test_is_mount_root(self): self.assertTrue(R.is_mount()) self.assertFalse((R / '\udfff').is_mount()) + def test_samefile(self): + parser = self.parser + fileA_path = parser.join(self.base, 'fileA') + fileB_path = parser.join(self.base, 'dirB', 'fileB') + p = self.cls(fileA_path) + pp = self.cls(fileA_path) + q = self.cls(fileB_path) + self.assertTrue(p.samefile(fileA_path)) + self.assertTrue(p.samefile(pp)) + self.assertFalse(p.samefile(fileB_path)) + self.assertFalse(p.samefile(q)) + # Test the non-existent file case + non_existent = parser.join(self.base, 'foo') + r = self.cls(non_existent) + self.assertRaises(FileNotFoundError, p.samefile, r) + self.assertRaises(FileNotFoundError, p.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, p) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + self.assertRaises(FileNotFoundError, r.samefile, r) + self.assertRaises(FileNotFoundError, r.samefile, non_existent) + def test_passing_kwargs_errors(self): with self.assertRaises(TypeError): self.cls(foo="bar") + @needs_symlinks + def test_iterdir_symlink(self): + # __iter__ on a symlink to a directory. + P = self.cls + p = P(self.base, 'linkB') + paths = set(p.iterdir()) + expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } + self.assertEqual(paths, expected) + def test_glob_empty_pattern(self): p = self.cls('') with self.assertRaisesRegex(ValueError, 'Unacceptable pattern'): @@ -1485,6 +2004,25 @@ def test_glob_dot(self): self.assertEqual( set(P('.').glob('**/*/*')), {P("dirD/fileD")}) + # See https://github.com/WebAssembly/wasi-filesystem/issues/26 + @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") + def test_glob_dotdot(self): + # ".." is not special in globs. + P = self.cls + p = P(self.base) + self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) + self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) + self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) + self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) + self.assertEqual(set(p.glob("dirA/../file*/..")), set()) + self.assertEqual(set(p.glob("../xyzzy")), set()) + if self.cls.parser is posixpath: + self.assertEqual(set(p.glob("xyzzy/..")), set()) + else: + # ".." segments are normalized first on Windows, so this path is stat()able. + self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) + self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + def test_glob_inaccessible(self): P = self.cls p = P(self.base, "mydir1", "mydir2") @@ -1500,6 +2038,124 @@ def test_rglob_pathlike(self): self.assertEqual(expect, set(p.rglob(P(pattern)))) self.assertEqual(expect, set(p.rglob(FakePath(pattern)))) + @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") + def test_glob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.glob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", []) + _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) + _check(p, "*A", ["dirA", "fileA", "linkA"]) + _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) + _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) + _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) + _check(p, "dir*/**", [ + "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", + "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", + "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", + "dirE/"]) + _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/"]) + _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", + "dirB/linkD/..", "dirA/linkC/linkD/..", + "dirC/..", "dirC/dirD/..", "dirE/.."]) + _check(p, "dir*/*/**", [ + "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", + "dirB/linkD/", "dirB/linkD/fileB", + "dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) + _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", + "dirB/linkD/..", "dirC/dirD/.."]) + _check(p, "dir*/**/fileC", ["dirC/fileC"]) + _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) + _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) + _check(p, "*/dirD/**/", ["dirC/dirD/"]) + + @needs_symlinks + @unittest.skipIf(is_emscripten, "Hangs") + def test_rglob_recurse_symlinks_common(self): + def _check(path, glob, expected): + actual = {path for path in path.rglob(glob, recurse_symlinks=True) + if path.parts.count("linkD") <= 1} # exclude symlink loop. + self.assertEqual(actual, { P(self.base, q) for q in expected }) + P = self.cls + p = P(self.base) + _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "*/fileA", []) + _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) + _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", + "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", + "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) + _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) + _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", + "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) + + p = P(self.base, "dirC") + _check(p, "*", ["dirC/fileC", "dirC/novel.txt", + "dirC/dirD", "dirC/dirD/fileD"]) + _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) + _check(p, "*/*", ["dirC/dirD/fileD"]) + _check(p, "*/", ["dirC/dirD/"]) + _check(p, "", ["dirC/", "dirC/dirD/"]) + # gh-91616, a re module regression + _check(p, "*.txt", ["dirC/novel.txt"]) + _check(p, "*.*", ["dirC/novel.txt"]) + + @needs_symlinks + def test_rglob_symlink_loop(self): + # Don't get fooled by symlink loops (Issue #26012). + P = self.cls + p = P(self.base) + given = set(p.rglob('*', recurse_symlinks=False)) + expect = {'brokenLink', + 'dirA', 'dirA/linkC', + 'dirB', 'dirB/fileB', 'dirB/linkD', + 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', + 'dirC/fileC', 'dirC/novel.txt', + 'dirE', + 'fileA', + 'linkA', + 'linkB', + 'brokenLinkLoop', + } + self.assertEqual(given, {p / x for x in expect}) + + @needs_symlinks + def test_glob_permissions(self): + # See bpo-38894 + P = self.cls + base = P(self.base) / 'permissions' + base.mkdir() + + for i in range(100): + link = base / f"link{i}" + if i % 2: + link.symlink_to(P(self.base, "dirE", "nonexistent")) + else: + link.symlink_to(P(self.base, "dirC"), target_is_directory=True) + + self.assertEqual(len(set(base.glob("*"))), 100) + self.assertEqual(len(set(base.glob("*/"))), 50) + self.assertEqual(len(set(base.glob("*/fileC"))), 50) + self.assertEqual(len(set(base.glob("*/file*"))), 50) + + @needs_symlinks + def test_glob_long_symlink(self): + # See gh-87695 + base = self.cls(self.base) / 'long_symlink' + base.mkdir() + bad_link = base / 'bad_link' + bad_link.symlink_to("bad" * 200) + self.assertEqual(sorted(base.glob('**/*')), [bad_link]) + @needs_posix def test_absolute_posix(self): P = self.cls @@ -1814,6 +2470,9 @@ class PathWalkTest(test_pathlib_abc.DummyPathWalkTest): can_symlink = PathTest.can_symlink def setUp(self): + name = self.id().split('.')[-1] + if name in _tests_needing_symlinks and not self.can_symlink: + self.skipTest('requires symlinks') super().setUp() sub21_path= self.sub2_path / "SUB21" tmp5_path = sub21_path / "tmp3" @@ -1895,6 +2554,37 @@ def test_walk_above_recursion_limit(self): list(base.walk()) list(base.walk(top_down=False)) + @needs_symlinks + def test_walk_follow_symlinks(self): + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with follow_symlinks=True") + + @needs_symlinks + def test_walk_symlink_location(self): + # Tests whether symlinks end up in filenames or dirnames depending + # on the `follow_symlinks` argument. + walk_it = self.walk_path.walk(follow_symlinks=False) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", files) + break + else: + self.fail("symlink not found") + + walk_it = self.walk_path.walk(follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.sub2_path: + self.assertIn("link", dirs) + break + else: + self.fail("symlink not found") + @unittest.skipIf(os.name == 'nt', 'test requires a POSIX-compatible system') class PosixPathTest(PathTest, PurePosixPathTest): diff --git a/Lib/test/test_pathlib/test_pathlib_abc.py b/Lib/test/test_pathlib/test_pathlib_abc.py index aaa30a17f2af14..e230dd188799a5 100644 --- a/Lib/test/test_pathlib/test_pathlib_abc.py +++ b/Lib/test/test_pathlib/test_pathlib_abc.py @@ -1,21 +1,19 @@ import collections -import contextlib import io import os import errno import stat import unittest -from pathlib._abc import UnsupportedOperation, ParserBase, PurePathBase, PathBase +from pathlib._abc import PurePathBase, PathBase +from pathlib._types import Parser import posixpath -from test.support import is_wasi from test.support.os_helper import TESTFN _tests_needing_posix = set() _tests_needing_windows = set() -_tests_needing_symlinks = set() def needs_posix(fn): @@ -28,32 +26,6 @@ def needs_windows(fn): _tests_needing_windows.add(fn.__name__) return fn -def needs_symlinks(fn): - """Decorator that marks a test as requiring a path class that supports symlinks.""" - _tests_needing_symlinks.add(fn.__name__) - return fn - - -class UnsupportedOperationTest(unittest.TestCase): - def test_is_notimplemented(self): - self.assertTrue(issubclass(UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(UnsupportedOperation(), NotImplementedError)) - - -class ParserBaseTest(unittest.TestCase): - cls = ParserBase - - def test_unsupported_operation(self): - m = self.cls() - e = UnsupportedOperation - with self.assertRaises(e): - m.sep - self.assertRaises(e, m.join, 'foo') - self.assertRaises(e, m.split, 'foo') - self.assertRaises(e, m.splitdrive, 'foo') - self.assertRaises(e, m.splitext, 'foo') - self.assertRaises(e, m.normcase, 'foo') - self.assertRaises(e, m.isabs, 'foo') # # Tests for the pure classes. @@ -63,37 +35,6 @@ def test_unsupported_operation(self): class PurePathBaseTest(unittest.TestCase): cls = PurePathBase - def test_unsupported_operation_pure(self): - p = self.cls('foo') - e = UnsupportedOperation - with self.assertRaises(e): - p.drive - with self.assertRaises(e): - p.root - with self.assertRaises(e): - p.anchor - with self.assertRaises(e): - p.parts - with self.assertRaises(e): - p.parent - with self.assertRaises(e): - p.parents - with self.assertRaises(e): - p.name - with self.assertRaises(e): - p.stem - with self.assertRaises(e): - p.suffix - with self.assertRaises(e): - p.suffixes - self.assertRaises(e, p.with_name, 'bar') - self.assertRaises(e, p.with_stem, 'bar') - self.assertRaises(e, p.with_suffix, '.txt') - self.assertRaises(e, p.relative_to, '') - self.assertRaises(e, p.is_relative_to, '') - self.assertRaises(e, p.is_absolute) - self.assertRaises(e, p.match, '*') - def test_magic_methods(self): P = self.cls self.assertFalse(hasattr(P, '__fspath__')) @@ -108,12 +49,11 @@ def test_magic_methods(self): self.assertIs(P.__ge__, object.__ge__) def test_parser(self): - self.assertIsInstance(self.cls.parser, ParserBase) + self.assertIs(self.cls.parser, posixpath) class DummyPurePath(PurePathBase): __slots__ = () - parser = posixpath def __eq__(self, other): if not isinstance(other, DummyPurePath): @@ -144,6 +84,9 @@ def setUp(self): self.sep = self.parser.sep self.altsep = self.parser.altsep + def test_parser(self): + self.assertIsInstance(self.cls.parser, Parser) + def test_constructor_common(self): P = self.cls p = P('a') @@ -1346,53 +1289,26 @@ def test_is_absolute_windows(self): class PathBaseTest(PurePathBaseTest): cls = PathBase - def test_unsupported_operation(self): - P = self.cls + def test_not_implemented_error(self): p = self.cls('') - e = UnsupportedOperation + e = NotImplementedError self.assertRaises(e, p.stat) - self.assertRaises(e, p.lstat) self.assertRaises(e, p.exists) - self.assertRaises(e, p.samefile, 'foo') self.assertRaises(e, p.is_dir) self.assertRaises(e, p.is_file) - self.assertRaises(e, p.is_mount) self.assertRaises(e, p.is_symlink) - self.assertRaises(e, p.is_block_device) - self.assertRaises(e, p.is_char_device) - self.assertRaises(e, p.is_fifo) - self.assertRaises(e, p.is_socket) self.assertRaises(e, p.open) self.assertRaises(e, p.read_bytes) self.assertRaises(e, p.read_text) self.assertRaises(e, p.write_bytes, b'foo') self.assertRaises(e, p.write_text, 'foo') self.assertRaises(e, p.iterdir) - self.assertRaises(e, p.glob, '*') - self.assertRaises(e, p.rglob, '*') + self.assertRaises(e, lambda: list(p.glob('*'))) + self.assertRaises(e, lambda: list(p.rglob('*'))) self.assertRaises(e, lambda: list(p.walk())) - self.assertRaises(e, p.absolute) - self.assertRaises(e, P.cwd) - self.assertRaises(e, p.expanduser) - self.assertRaises(e, p.home) self.assertRaises(e, p.readlink) self.assertRaises(e, p.symlink_to, 'foo') - self.assertRaises(e, p.hardlink_to, 'foo') self.assertRaises(e, p.mkdir) - self.assertRaises(e, p.touch) - self.assertRaises(e, p.rename, 'foo') - self.assertRaises(e, p.replace, 'foo') - self.assertRaises(e, p.chmod, 0o755) - self.assertRaises(e, p.lchmod, 0o755) - self.assertRaises(e, p.unlink) - self.assertRaises(e, p.rmdir) - self.assertRaises(e, p.owner) - self.assertRaises(e, p.group) - self.assertRaises(e, p.as_uri) - - def test_as_uri_common(self): - e = UnsupportedOperation - self.assertRaises(e, self.cls('').as_uri) def test_fspath_common(self): self.assertRaises(TypeError, os.fspath, self.cls('')) @@ -1421,35 +1337,15 @@ def close(self): 'st_mode st_ino st_dev st_nlink st_uid st_gid st_size st_atime st_mtime st_ctime') -class DummyDirEntry: - """ - Minimal os.DirEntry-like object. Returned from DummyPath.scandir(). - """ - __slots__ = ('name', '_is_symlink', '_is_dir') - - def __init__(self, name, is_symlink, is_dir): - self.name = name - self._is_symlink = is_symlink - self._is_dir = is_dir - - def is_symlink(self): - return self._is_symlink - - def is_dir(self, *, follow_symlinks=True): - return self._is_dir and (follow_symlinks or not self._is_symlink) - - class DummyPath(PathBase): """ Simple implementation of PathBase that keeps files and directories in memory. """ __slots__ = () - parser = posixpath _files = {} _directories = {} - _symlinks = {} def __eq__(self, other): if not isinstance(other, DummyPath): @@ -1463,16 +1359,11 @@ def __repr__(self): return "{}({!r})".format(self.__class__.__name__, self.as_posix()) def stat(self, *, follow_symlinks=True): - if follow_symlinks or self.name in ('', '.', '..'): - path = str(self.resolve(strict=True)) - else: - path = str(self.parent.resolve(strict=True) / self.name) + path = str(self).rstrip('/') if path in self._files: st_mode = stat.S_IFREG elif path in self._directories: st_mode = stat.S_IFDIR - elif path in self._symlinks: - st_mode = stat.S_IFLNK else: raise FileNotFoundError(errno.ENOENT, "Not found", str(self)) return DummyPathStatResult(st_mode, hash(str(self)), 0, 0, 0, 0, 0, 0, 0, 0) @@ -1481,10 +1372,7 @@ def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): if buffering != -1 and not (buffering == 0 and 'b' in mode): raise NotImplementedError - path_obj = self.resolve() - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) + path = str(self) if path in self._directories: raise IsADirectoryError(errno.EISDIR, "Is a directory", path) @@ -1495,6 +1383,7 @@ def open(self, mode='r', buffering=-1, encoding=None, raise FileNotFoundError(errno.ENOENT, "File not found", path) stream = io.BytesIO(self._files[path]) elif mode == 'w': + parent, name = posixpath.split(path) if parent not in self._directories: raise FileNotFoundError(errno.ENOENT, "File not found", parent) stream = DummyPathIO(self._files, path) @@ -1506,30 +1395,19 @@ def open(self, mode='r', buffering=-1, encoding=None, stream = io.TextIOWrapper(stream, encoding=encoding, errors=errors, newline=newline) return stream - @contextlib.contextmanager - def scandir(self): - path = self.resolve() - path_str = str(path) - if path_str in self._files: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path_str) - elif path_str in self._directories: - yield iter([path.joinpath(name)._dir_entry for name in self._directories[path_str]]) + def iterdir(self): + path = str(self).rstrip('/') + if path in self._files: + raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) + elif path in self._directories: + return iter([self / name for name in self._directories[path]]) else: - raise FileNotFoundError(errno.ENOENT, "File not found", path_str) - - @property - def _dir_entry(self): - path_str = str(self) - is_symlink = path_str in self._symlinks - is_directory = (path_str in self._directories - if not is_symlink - else self._symlinks[path_str][1]) - return DummyDirEntry(self.name, is_symlink, is_directory) + raise FileNotFoundError(errno.ENOENT, "File not found", path) def mkdir(self, mode=0o777, parents=False, exist_ok=False): - path = str(self.parent.resolve() / self.name) - parent = str(self.parent.resolve()) - if path in self._directories or path in self._symlinks: + path = str(self) + parent = str(self.parent) + if path in self._directories: if exist_ok: return else: @@ -1544,36 +1422,18 @@ def mkdir(self, mode=0o777, parents=False, exist_ok=False): self.parent.mkdir(parents=True, exist_ok=True) self.mkdir(mode, parents=False, exist_ok=exist_ok) - def unlink(self, missing_ok=False): - path_obj = self.parent.resolve(strict=True) / self.name - path = str(path_obj) - name = path_obj.name - parent = str(path_obj.parent) - if path in self._directories: - raise IsADirectoryError(errno.EISDIR, "Is a directory", path) - elif path in self._files: - self._directories[parent].remove(name) + def _delete(self): + path = str(self) + if path in self._files: del self._files[path] - elif path in self._symlinks: - self._directories[parent].remove(name) - del self._symlinks[path] - elif not missing_ok: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def rmdir(self): - path_obj = self.parent.resolve(strict=True) / self.name - path = str(path_obj) - if path in self._files or path in self._symlinks: - raise NotADirectoryError(errno.ENOTDIR, "Not a directory", path) - elif path not in self._directories: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - elif self._directories[path]: - raise OSError(errno.ENOTEMPTY, "Directory not empty", path) - else: - name = path_obj.name - parent = str(path_obj.parent) - self._directories[parent].remove(name) + elif path in self._directories: + for name in list(self._directories[path]): + self.joinpath(name)._delete() del self._directories[path] + else: + raise FileNotFoundError(errno.ENOENT, "File not found", path) + parent = str(self.parent) + self._directories[parent].remove(self.name) class DummyPathTest(DummyPurePathTest): @@ -1604,9 +1464,6 @@ class DummyPathTest(DummyPurePathTest): def setUp(self): super().setUp() - name = self.id().split('.')[-1] - if name in _tests_needing_symlinks and not self.can_symlink: - self.skipTest('requires symlinks') parser = self.cls.parser p = self.cls(self.base) p.mkdir(parents=True) @@ -1639,7 +1496,6 @@ def tearDown(self): cls = self.cls cls._files.clear() cls._directories.clear() - cls._symlinks.clear() def tempdir(self): path = self.cls(self.base).with_name('tmp-dirD') @@ -1655,27 +1511,6 @@ def assertEqualNormCase(self, path_a, path_b): normcase = self.parser.normcase self.assertEqual(normcase(path_a), normcase(path_b)) - def test_samefile(self): - parser = self.parser - fileA_path = parser.join(self.base, 'fileA') - fileB_path = parser.join(self.base, 'dirB', 'fileB') - p = self.cls(fileA_path) - pp = self.cls(fileA_path) - q = self.cls(fileB_path) - self.assertTrue(p.samefile(fileA_path)) - self.assertTrue(p.samefile(pp)) - self.assertFalse(p.samefile(fileB_path)) - self.assertFalse(p.samefile(q)) - # Test the non-existent file case - non_existent = parser.join(self.base, 'foo') - r = self.cls(non_existent) - self.assertRaises(FileNotFoundError, p.samefile, r) - self.assertRaises(FileNotFoundError, p.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, p) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - self.assertRaises(FileNotFoundError, r.samefile, r) - self.assertRaises(FileNotFoundError, r.samefile, non_existent) - def test_exists(self): P = self.cls p = P(self.base) @@ -1765,101 +1600,6 @@ def test_copy_file(self): self.assertTrue(target.exists()) self.assertEqual(source.read_text(), target.read_text()) - @needs_symlinks - def test_copy_symlink_follow_symlinks_true(self): - base = self.cls(self.base) - source = base / 'linkA' - target = base / 'copyA' - result = source.copy(target) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertFalse(target.is_symlink()) - self.assertEqual(source.read_text(), target.read_text()) - - @needs_symlinks - def test_copy_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'linkA' - target = base / 'copyA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source.readlink(), target.readlink()) - - @needs_symlinks - def test_copy_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkA' - self.assertRaises(OSError, source.copy, source) - - @needs_symlinks - def test_copy_symlink_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'fileA') - target.symlink_to(base / 'dirC') - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_symlink_to_existing_directory_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'fileA') - target.symlink_to(base / 'dirC') - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'linkB' - target = base / 'copyA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source.readlink(), target.readlink()) - - @needs_symlinks - def test_copy_directory_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - self.assertRaises(OSError, source.copy, source) - self.assertRaises(OSError, source.copy, source, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_into_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - target = base / 'linkB' / 'copyB' - self.assertRaises(OSError, source.copy, target) - self.assertRaises(OSError, source.copy, target, follow_symlinks=False) - self.assertFalse(target.exists()) - - @needs_symlinks - def test_copy_directory_symlink_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'dirC') - target.symlink_to(base / 'fileA') - self.assertRaises(FileExistsError, source.copy, target) - self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) - - @needs_symlinks - def test_copy_directory_symlink_to_existing_directory_symlink(self): - base = self.cls(self.base) - source = base / 'copySource' - target = base / 'copyTarget' - source.symlink_to(base / 'dirC' / 'dirD') - target.symlink_to(base / 'dirC') - self.assertRaises(FileExistsError, source.copy, target) - self.assertRaises(FileExistsError, source.copy, target, follow_symlinks=False) - def test_copy_file_to_existing_file(self): base = self.cls(self.base) source = base / 'fileA' @@ -1875,34 +1615,6 @@ def test_copy_file_to_existing_directory(self): target = base / 'dirA' self.assertRaises(OSError, source.copy, target) - @needs_symlinks - def test_copy_file_to_existing_symlink(self): - base = self.cls(self.base) - source = base / 'dirB' / 'fileB' - target = base / 'linkA' - real_target = base / 'fileA' - result = source.copy(target) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertTrue(real_target.exists()) - self.assertFalse(real_target.is_symlink()) - self.assertEqual(source.read_text(), real_target.read_text()) - - @needs_symlinks - def test_copy_file_to_existing_symlink_follow_symlinks_false(self): - base = self.cls(self.base) - source = base / 'dirB' / 'fileB' - target = base / 'linkA' - real_target = base / 'fileA' - result = source.copy(target, follow_symlinks=False) - self.assertEqual(result, target) - self.assertTrue(target.exists()) - self.assertTrue(target.is_symlink()) - self.assertTrue(real_target.exists()) - self.assertFalse(real_target.is_symlink()) - self.assertEqual(source.read_text(), real_target.read_text()) - def test_copy_file_empty(self): base = self.cls(self.base) source = base / 'empty' @@ -2020,23 +1732,6 @@ def test_copy_dir_into_itself(self): self.assertRaises(OSError, source.copy, target, follow_symlinks=False) self.assertFalse(target.exists()) - @needs_symlinks - def test_copy_dangling_symlink(self): - base = self.cls(self.base) - source = base / 'source' - target = base / 'target' - - source.mkdir() - source.joinpath('link').symlink_to('nonexistent') - - self.assertRaises(FileNotFoundError, source.copy, target) - - target2 = base / 'target2' - result = source.copy(target2, follow_symlinks=False) - self.assertEqual(result, target2) - self.assertTrue(target2.joinpath('link').is_symlink()) - self.assertEqual(target2.joinpath('link').readlink(), self.cls('nonexistent')) - def test_copy_into(self): base = self.cls(self.base) source = base / 'fileA' @@ -2122,54 +1817,6 @@ def test_move_dir_into_itself(self): self.assertTrue(source.exists()) self.assertFalse(target.exists()) - @needs_symlinks - def test_move_file_symlink(self): - base = self.cls(self.base) - source = base / 'linkA' - source_readlink = source.readlink() - target = base / 'linkA_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - - @needs_symlinks - def test_move_file_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkA' - self.assertRaises(OSError, source.move, source) - - @needs_symlinks - def test_move_dir_symlink(self): - base = self.cls(self.base) - source = base / 'linkB' - source_readlink = source.readlink() - target = base / 'linkB_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - - @needs_symlinks - def test_move_dir_symlink_to_itself(self): - base = self.cls(self.base) - source = base / 'linkB' - self.assertRaises(OSError, source.move, source) - - @needs_symlinks - def test_move_dangling_symlink(self): - base = self.cls(self.base) - source = base / 'brokenLink' - source_readlink = source.readlink() - target = base / 'brokenLink_moved' - result = source.move(target) - self.assertEqual(result, target) - self.assertFalse(source.exists()) - self.assertTrue(target.is_symlink()) - self.assertEqual(source_readlink, target.readlink()) - def test_move_into(self): base = self.cls(self.base) source = base / 'fileA' @@ -2196,15 +1843,6 @@ def test_iterdir(self): expected += ['linkA', 'linkB', 'brokenLink', 'brokenLinkLoop'] self.assertEqual(paths, { P(self.base, q) for q in expected }) - @needs_symlinks - def test_iterdir_symlink(self): - # __iter__ on a symlink to a directory. - P = self.cls - p = P(self.base, 'linkB') - paths = set(p.iterdir()) - expected = { P(self.base, 'linkB', q) for q in ['fileB', 'linkD'] } - self.assertEqual(paths, expected) - def test_iterdir_nodir(self): # __iter__ on something that is not a directory. p = self.cls(self.base, 'fileA') @@ -2217,9 +1855,9 @@ def test_iterdir_nodir(self): def test_scandir(self): p = self.cls(self.base) - with p.scandir() as entries: + with p._scandir() as entries: self.assertTrue(list(entries)) - with p.scandir() as entries: + with p._scandir() as entries: for entry in entries: child = p / entry.name self.assertIsNotNone(entry) @@ -2231,7 +1869,6 @@ def test_scandir(self): if entry.name != 'brokenLinkLoop': self.assertEqual(entry.is_dir(), child.is_dir()) - def test_glob_common(self): def _check(glob, expected): self.assertEqual(set(glob), { P(self.base, q) for q in expected }) @@ -2285,8 +1922,6 @@ def test_glob_empty_pattern(self): P = self.cls p = P(self.base) self.assertEqual(list(p.glob("")), [p]) - self.assertEqual(list(p.glob(".")), [p / "."]) - self.assertEqual(list(p.glob("./")), [p / "./"]) def test_glob_case_sensitive(self): P = self.cls @@ -2300,43 +1935,6 @@ def _check(path, pattern, case_sensitive, expected): _check(path, "dirb/file*", True, []) _check(path, "dirb/file*", False, ["dirB/fileB"]) - @needs_symlinks - def test_glob_recurse_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.glob(glob, recurse_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", []) - _check(p, "dir*/file*", ["dirB/fileB", "dirC/fileC"]) - _check(p, "*A", ["dirA", "fileA", "linkA"]) - _check(p, "*B/*", ["dirB/fileB", "dirB/linkD", "linkB/fileB", "linkB/linkD"]) - _check(p, "*/fileB", ["dirB/fileB", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirB/", "dirC/", "dirE/", "linkB/"]) - _check(p, "dir*/*/..", ["dirC/dirD/..", "dirA/linkC/..", "dirB/linkD/.."]) - _check(p, "dir*/**", [ - "dirA/", "dirA/linkC", "dirA/linkC/fileB", "dirA/linkC/linkD", "dirA/linkC/linkD/fileB", - "dirB/", "dirB/fileB", "dirB/linkD", "dirB/linkD/fileB", - "dirC/", "dirC/fileC", "dirC/dirD", "dirC/dirD/fileD", "dirC/novel.txt", - "dirE/"]) - _check(p, "dir*/**/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/"]) - _check(p, "dir*/**/..", ["dirA/..", "dirA/linkC/..", "dirB/..", - "dirB/linkD/..", "dirA/linkC/linkD/..", - "dirC/..", "dirC/dirD/..", "dirE/.."]) - _check(p, "dir*/*/**", [ - "dirA/linkC/", "dirA/linkC/linkD", "dirA/linkC/fileB", "dirA/linkC/linkD/fileB", - "dirB/linkD/", "dirB/linkD/fileB", - "dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "dir*/*/**/", ["dirA/linkC/", "dirA/linkC/linkD/", "dirB/linkD/", "dirC/dirD/"]) - _check(p, "dir*/*/**/..", ["dirA/linkC/..", "dirA/linkC/linkD/..", - "dirB/linkD/..", "dirC/dirD/.."]) - _check(p, "dir*/**/fileC", ["dirC/fileC"]) - _check(p, "dir*/*/../dirD/**/", ["dirC/dirD/../dirD/"]) - _check(p, "*/dirD/**", ["dirC/dirD/", "dirC/dirD/fileD"]) - _check(p, "*/dirD/**/", ["dirC/dirD/"]) - def test_rglob_recurse_symlinks_false(self): def _check(path, glob, expected): actual = set(path.rglob(glob, recurse_symlinks=False)) @@ -2395,251 +1993,6 @@ def test_rglob_windows(self): self.assertEqual(set(p.rglob("FILEd")), { P(self.base, "dirC/dirD/fileD") }) self.assertEqual(set(p.rglob("*\\")), { P(self.base, "dirC/dirD/") }) - @needs_symlinks - def test_rglob_recurse_symlinks_common(self): - def _check(path, glob, expected): - actual = {path for path in path.rglob(glob, recurse_symlinks=True) - if path.parts.count("linkD") <= 1} # exclude symlink loop. - self.assertEqual(actual, { P(self.base, q) for q in expected }) - P = self.cls - p = P(self.base) - _check(p, "fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "*/fileA", []) - _check(p, "*/fileB", ["dirB/fileB", "dirA/linkC/fileB", "linkB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB"]) - _check(p, "file*", ["fileA", "dirA/linkC/fileB", "dirB/fileB", - "dirA/linkC/linkD/fileB", "dirB/linkD/fileB", "linkB/linkD/fileB", - "dirC/fileC", "dirC/dirD/fileD", "linkB/fileB"]) - _check(p, "*/", ["dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirC/dirD/", "dirE/", "linkB/", "linkB/linkD/"]) - _check(p, "", ["", "dirA/", "dirA/linkC/", "dirA/linkC/linkD/", "dirB/", "dirB/linkD/", - "dirC/", "dirE/", "dirC/dirD/", "linkB/", "linkB/linkD/"]) - - p = P(self.base, "dirC") - _check(p, "*", ["dirC/fileC", "dirC/novel.txt", - "dirC/dirD", "dirC/dirD/fileD"]) - _check(p, "file*", ["dirC/fileC", "dirC/dirD/fileD"]) - _check(p, "*/*", ["dirC/dirD/fileD"]) - _check(p, "*/", ["dirC/dirD/"]) - _check(p, "", ["dirC/", "dirC/dirD/"]) - # gh-91616, a re module regression - _check(p, "*.txt", ["dirC/novel.txt"]) - _check(p, "*.*", ["dirC/novel.txt"]) - - @needs_symlinks - def test_rglob_symlink_loop(self): - # Don't get fooled by symlink loops (Issue #26012). - P = self.cls - p = P(self.base) - given = set(p.rglob('*', recurse_symlinks=False)) - expect = {'brokenLink', - 'dirA', 'dirA/linkC', - 'dirB', 'dirB/fileB', 'dirB/linkD', - 'dirC', 'dirC/dirD', 'dirC/dirD/fileD', - 'dirC/fileC', 'dirC/novel.txt', - 'dirE', - 'fileA', - 'linkA', - 'linkB', - 'brokenLinkLoop', - } - self.assertEqual(given, {p / x for x in expect}) - - # See https://github.com/WebAssembly/wasi-filesystem/issues/26 - @unittest.skipIf(is_wasi, "WASI resolution of '..' parts doesn't match POSIX") - def test_glob_dotdot(self): - # ".." is not special in globs. - P = self.cls - p = P(self.base) - self.assertEqual(set(p.glob("..")), { P(self.base, "..") }) - self.assertEqual(set(p.glob("../..")), { P(self.base, "..", "..") }) - self.assertEqual(set(p.glob("dirA/..")), { P(self.base, "dirA", "..") }) - self.assertEqual(set(p.glob("dirA/../file*")), { P(self.base, "dirA/../fileA") }) - self.assertEqual(set(p.glob("dirA/../file*/..")), set()) - self.assertEqual(set(p.glob("../xyzzy")), set()) - if self.cls.parser is posixpath: - self.assertEqual(set(p.glob("xyzzy/..")), set()) - else: - # ".." segments are normalized first on Windows, so this path is stat()able. - self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) - - @needs_symlinks - def test_glob_permissions(self): - # See bpo-38894 - P = self.cls - base = P(self.base) / 'permissions' - base.mkdir() - - for i in range(100): - link = base / f"link{i}" - if i % 2: - link.symlink_to(P(self.base, "dirE", "nonexistent")) - else: - link.symlink_to(P(self.base, "dirC"), target_is_directory=True) - - self.assertEqual(len(set(base.glob("*"))), 100) - self.assertEqual(len(set(base.glob("*/"))), 50) - self.assertEqual(len(set(base.glob("*/fileC"))), 50) - self.assertEqual(len(set(base.glob("*/file*"))), 50) - - @needs_symlinks - def test_glob_long_symlink(self): - # See gh-87695 - base = self.cls(self.base) / 'long_symlink' - base.mkdir() - bad_link = base / 'bad_link' - bad_link.symlink_to("bad" * 200) - self.assertEqual(sorted(base.glob('**/*')), [bad_link]) - - @needs_posix - def test_absolute_posix(self): - P = self.cls - # The default implementation uses '/' as the current directory - self.assertEqual(str(P('').absolute()), '/') - self.assertEqual(str(P('a').absolute()), '/a') - self.assertEqual(str(P('a/b').absolute()), '/a/b') - - self.assertEqual(str(P('/').absolute()), '/') - self.assertEqual(str(P('/a').absolute()), '/a') - self.assertEqual(str(P('/a/b').absolute()), '/a/b') - - # '//'-prefixed absolute path (supported by POSIX). - self.assertEqual(str(P('//').absolute()), '//') - self.assertEqual(str(P('//a').absolute()), '//a') - self.assertEqual(str(P('//a/b').absolute()), '//a/b') - - @needs_symlinks - def test_readlink(self): - P = self.cls(self.base) - self.assertEqual((P / 'linkA').readlink(), self.cls('fileA')) - self.assertEqual((P / 'brokenLink').readlink(), - self.cls('non-existing')) - self.assertEqual((P / 'linkB').readlink(), self.cls('dirB')) - self.assertEqual((P / 'linkB' / 'linkD').readlink(), self.cls('../dirB')) - with self.assertRaises(OSError): - (P / 'fileA').readlink() - - @unittest.skipIf(hasattr(os, "readlink"), "os.readlink() is present") - def test_readlink_unsupported(self): - P = self.cls(self.base) - p = P / 'fileA' - with self.assertRaises(UnsupportedOperation): - q.readlink(p) - - def _check_resolve(self, p, expected, strict=True): - q = p.resolve(strict) - self.assertEqual(q, expected) - - # This can be used to check both relative and absolute resolutions. - _check_resolve_relative = _check_resolve_absolute = _check_resolve - - @needs_symlinks - def test_resolve_common(self): - P = self.cls - p = P(self.base, 'foo') - with self.assertRaises(OSError) as cm: - p.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ENOENT) - # Non-strict - parser = self.parser - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(self.base, 'foo')) - p = P(self.base, 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(self.base, 'foo', 'in', 'spam')) - p = P(self.base, '..', 'foo', 'in', 'spam') - self.assertEqualNormCase(str(p.resolve(strict=False)), - parser.join(parser.dirname(self.base), 'foo', 'in', 'spam')) - # These are all relative symlinks. - p = P(self.base, 'dirB', 'fileB') - self._check_resolve_relative(p, p) - p = P(self.base, 'linkA') - self._check_resolve_relative(p, P(self.base, 'fileA')) - p = P(self.base, 'dirA', 'linkC', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - p = P(self.base, 'dirB', 'linkD', 'fileB') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkC', 'fileB', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'fileB', 'foo', 'in', - 'spam'), False) - p = P(self.base, 'dirA', 'linkC', '..', 'foo', 'in', 'spam') - if self.cls.parser is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(self.base, 'dirA', 'foo', 'in', - 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - # Now create absolute symlinks. - d = self.tempdir() - P(self.base, 'dirA', 'linkX').symlink_to(d) - P(self.base, str(d), 'linkY').symlink_to(self.parser.join(self.base, 'dirB')) - p = P(self.base, 'dirA', 'linkX', 'linkY', 'fileB') - self._check_resolve_absolute(p, P(self.base, 'dirB', 'fileB')) - # Non-strict - p = P(self.base, 'dirA', 'linkX', 'linkY', 'foo', 'in', 'spam') - self._check_resolve_relative(p, P(self.base, 'dirB', 'foo', 'in', 'spam'), - False) - p = P(self.base, 'dirA', 'linkX', 'linkY', '..', 'foo', 'in', 'spam') - if self.cls.parser is not posixpath: - # In Windows, if linkY points to dirB, 'dirA\linkY\..' - # resolves to 'dirA' without resolving linkY first. - self._check_resolve_relative(p, P(d, 'foo', 'in', 'spam'), False) - else: - # In Posix, if linkY points to dirB, 'dirA/linkY/..' - # resolves to 'dirB/..' first before resolving to parent of dirB. - self._check_resolve_relative(p, P(self.base, 'foo', 'in', 'spam'), False) - - @needs_symlinks - def test_resolve_dot(self): - # See http://web.archive.org/web/20200623062557/https://bitbucket.org/pitrou/pathlib/issues/9/ - parser = self.parser - p = self.cls(self.base) - p.joinpath('0').symlink_to('.', target_is_directory=True) - p.joinpath('1').symlink_to(parser.join('0', '0'), target_is_directory=True) - p.joinpath('2').symlink_to(parser.join('1', '1'), target_is_directory=True) - q = p / '2' - self.assertEqual(q.resolve(strict=True), p) - r = q / '3' / '4' - self.assertRaises(FileNotFoundError, r.resolve, strict=True) - # Non-strict - self.assertEqual(r.resolve(strict=False), p / '3' / '4') - - def _check_symlink_loop(self, *args): - path = self.cls(*args) - with self.assertRaises(OSError) as cm: - path.resolve(strict=True) - self.assertEqual(cm.exception.errno, errno.ELOOP) - - @needs_posix - @needs_symlinks - def test_resolve_loop(self): - # Loops with relative symlinks. - self.cls(self.base, 'linkX').symlink_to('linkX/inside') - self._check_symlink_loop(self.base, 'linkX') - self.cls(self.base, 'linkY').symlink_to('linkY') - self._check_symlink_loop(self.base, 'linkY') - self.cls(self.base, 'linkZ').symlink_to('linkZ/../linkZ') - self._check_symlink_loop(self.base, 'linkZ') - # Non-strict - p = self.cls(self.base, 'linkZ', 'foo') - self.assertEqual(p.resolve(strict=False), p) - # Loops with absolute symlinks. - self.cls(self.base, 'linkU').symlink_to(self.parser.join(self.base, 'linkU/inside')) - self._check_symlink_loop(self.base, 'linkU') - self.cls(self.base, 'linkV').symlink_to(self.parser.join(self.base, 'linkV')) - self._check_symlink_loop(self.base, 'linkV') - self.cls(self.base, 'linkW').symlink_to(self.parser.join(self.base, 'linkW/../linkW')) - self._check_symlink_loop(self.base, 'linkW') - # Non-strict - q = self.cls(self.base, 'linkW', 'foo') - self.assertEqual(q.resolve(strict=False), q) - def test_stat(self): statA = self.cls(self.base).joinpath('fileA').stat() statB = self.cls(self.base).joinpath('dirB', 'fileB').stat() @@ -2660,28 +2013,11 @@ def test_stat(self): self.assertEqual(statA.st_dev, statC.st_dev) # other attributes not used by pathlib. - @needs_symlinks - def test_stat_no_follow_symlinks(self): - p = self.cls(self.base) / 'linkA' - st = p.stat() - self.assertNotEqual(st, p.stat(follow_symlinks=False)) - def test_stat_no_follow_symlinks_nosymlink(self): p = self.cls(self.base) / 'fileA' st = p.stat() self.assertEqual(st, p.stat(follow_symlinks=False)) - @needs_symlinks - def test_lstat(self): - p = self.cls(self.base)/ 'linkA' - st = p.stat() - self.assertNotEqual(st, p.lstat()) - - def test_lstat_nosymlink(self): - p = self.cls(self.base) / 'fileA' - st = p.stat() - self.assertEqual(st, p.lstat()) - def test_is_dir(self): P = self.cls(self.base) self.assertTrue((P / 'dirA').is_dir()) @@ -2734,15 +2070,6 @@ def test_is_file_no_follow_symlinks(self): self.assertFalse((P / 'fileA\udfff').is_file(follow_symlinks=False)) self.assertFalse((P / 'fileA\x00').is_file(follow_symlinks=False)) - def test_is_mount(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_mount()) - self.assertFalse((P / 'dirA').is_mount()) - self.assertFalse((P / 'non-existing').is_mount()) - self.assertFalse((P / 'fileA' / 'bah').is_mount()) - if self.can_symlink: - self.assertFalse((P / 'linkA').is_mount()) - def test_is_symlink(self): P = self.cls(self.base) self.assertFalse((P / 'fileA').is_symlink()) @@ -2759,120 +2086,23 @@ def test_is_symlink(self): self.assertIs((P / 'linkA\udfff').is_file(), False) self.assertIs((P / 'linkA\x00').is_file(), False) - def test_is_junction_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_junction()) - self.assertFalse((P / 'dirA').is_junction()) - self.assertFalse((P / 'non-existing').is_junction()) - self.assertFalse((P / 'fileA' / 'bah').is_junction()) - self.assertFalse((P / 'fileA\udfff').is_junction()) - self.assertFalse((P / 'fileA\x00').is_junction()) - - def test_is_fifo_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_fifo()) - self.assertFalse((P / 'dirA').is_fifo()) - self.assertFalse((P / 'non-existing').is_fifo()) - self.assertFalse((P / 'fileA' / 'bah').is_fifo()) - self.assertIs((P / 'fileA\udfff').is_fifo(), False) - self.assertIs((P / 'fileA\x00').is_fifo(), False) - - def test_is_socket_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_socket()) - self.assertFalse((P / 'dirA').is_socket()) - self.assertFalse((P / 'non-existing').is_socket()) - self.assertFalse((P / 'fileA' / 'bah').is_socket()) - self.assertIs((P / 'fileA\udfff').is_socket(), False) - self.assertIs((P / 'fileA\x00').is_socket(), False) - - def test_is_block_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_block_device()) - self.assertFalse((P / 'dirA').is_block_device()) - self.assertFalse((P / 'non-existing').is_block_device()) - self.assertFalse((P / 'fileA' / 'bah').is_block_device()) - self.assertIs((P / 'fileA\udfff').is_block_device(), False) - self.assertIs((P / 'fileA\x00').is_block_device(), False) - - def test_is_char_device_false(self): - P = self.cls(self.base) - self.assertFalse((P / 'fileA').is_char_device()) - self.assertFalse((P / 'dirA').is_char_device()) - self.assertFalse((P / 'non-existing').is_char_device()) - self.assertFalse((P / 'fileA' / 'bah').is_char_device()) - self.assertIs((P / 'fileA\udfff').is_char_device(), False) - self.assertIs((P / 'fileA\x00').is_char_device(), False) - - def _check_complex_symlinks(self, link0_target): - # Test solving a non-looping chain of symlinks (issue #19887). - parser = self.parser - P = self.cls(self.base) - P.joinpath('link1').symlink_to(parser.join('link0', 'link0'), target_is_directory=True) - P.joinpath('link2').symlink_to(parser.join('link1', 'link1'), target_is_directory=True) - P.joinpath('link3').symlink_to(parser.join('link2', 'link2'), target_is_directory=True) - P.joinpath('link0').symlink_to(link0_target, target_is_directory=True) - - # Resolve absolute paths. - p = (P / 'link0').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link1').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link2').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - p = (P / 'link3').resolve() - self.assertEqual(p, P) - self.assertEqualNormCase(str(p), self.base) - - @needs_symlinks - def test_complex_symlinks_absolute(self): - self._check_complex_symlinks(self.base) - - @needs_symlinks - def test_complex_symlinks_relative(self): - self._check_complex_symlinks('.') - - @needs_symlinks - def test_complex_symlinks_relative_dot_dot(self): - self._check_complex_symlinks(self.parser.join('dirA', '..')) - - def test_unlink(self): - p = self.cls(self.base) / 'fileA' - p.unlink() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - - def test_unlink_missing_ok(self): - p = self.cls(self.base) / 'fileAAA' - self.assertFileNotFound(p.unlink) - p.unlink(missing_ok=True) - - def test_rmdir(self): - p = self.cls(self.base) / 'dirA' - for q in p.iterdir(): - q.unlink() - p.rmdir() - self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) - def test_delete_file(self): p = self.cls(self.base) / 'fileA' p._delete() self.assertFileNotFound(p.stat) - self.assertFileNotFound(p.unlink) + self.assertFileNotFound(p._delete) def test_delete_dir(self): base = self.cls(self.base) base.joinpath('dirA')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirA').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').lstat) + self.assertRaises(FileNotFoundError, base.joinpath('dirA', 'linkC').stat, + follow_symlinks=False) base.joinpath('dirB')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirB').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'fileB').stat) - self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').lstat) + self.assertRaises(FileNotFoundError, base.joinpath('dirB', 'linkD').stat, + follow_symlinks=False) base.joinpath('dirC')._delete() self.assertRaises(FileNotFoundError, base.joinpath('dirC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'dirD').stat) @@ -2880,41 +2110,6 @@ def test_delete_dir(self): self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'fileC').stat) self.assertRaises(FileNotFoundError, base.joinpath('dirC', 'novel.txt').stat) - @needs_symlinks - def test_delete_symlink(self): - tmp = self.cls(self.base, 'delete') - tmp.mkdir() - dir_ = tmp / 'dir' - dir_.mkdir() - link = tmp / 'link' - link.symlink_to(dir_) - link._delete() - self.assertTrue(dir_.exists()) - self.assertFalse(link.exists(follow_symlinks=False)) - - @needs_symlinks - def test_delete_inner_symlink(self): - tmp = self.cls(self.base, 'delete') - tmp.mkdir() - dir1 = tmp / 'dir1' - dir2 = dir1 / 'dir2' - dir3 = tmp / 'dir3' - for d in dir1, dir2, dir3: - d.mkdir() - file1 = tmp / 'file1' - file1.write_text('foo') - link1 = dir1 / 'link1' - link1.symlink_to(dir2) - link2 = dir1 / 'link2' - link2.symlink_to(dir3) - link3 = dir1 / 'link3' - link3.symlink_to(file1) - # make sure symlinks are removed but not followed - dir1._delete() - self.assertFalse(dir1.exists()) - self.assertTrue(dir3.exists()) - self.assertTrue(file1.exists()) - def test_delete_missing(self): tmp = self.cls(self.base, 'delete') tmp.mkdir() @@ -2929,9 +2124,6 @@ class DummyPathWalkTest(unittest.TestCase): can_symlink = False def setUp(self): - name = self.id().split('.')[-1] - if name in _tests_needing_symlinks and not self.can_symlink: - self.skipTest('requires symlinks') # Build: # TESTFN/ # TEST1/ a file kid and two directory kids @@ -2977,7 +2169,7 @@ def setUp(self): def tearDown(self): base = self.cls(self.base) - base._rmtree() + base._delete() def test_walk_topdown(self): walker = self.walk_path.walk() @@ -3044,70 +2236,6 @@ def test_walk_bottom_up(self): raise AssertionError(f"Unexpected path: {path}") self.assertTrue(seen_testfn) - @needs_symlinks - def test_walk_follow_symlinks(self): - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.link_path: - self.assertEqual(dirs, []) - self.assertEqual(files, ["tmp4"]) - break - else: - self.fail("Didn't follow symlink with follow_symlinks=True") - - @needs_symlinks - def test_walk_symlink_location(self): - # Tests whether symlinks end up in filenames or dirnames depending - # on the `follow_symlinks` argument. - walk_it = self.walk_path.walk(follow_symlinks=False) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", files) - break - else: - self.fail("symlink not found") - - walk_it = self.walk_path.walk(follow_symlinks=True) - for root, dirs, files in walk_it: - if root == self.sub2_path: - self.assertIn("link", dirs) - break - else: - self.fail("symlink not found") - - -class DummyPathWithSymlinks(DummyPath): - __slots__ = () - - # Reduce symlink traversal limit to make tests run faster. - _max_symlinks = 20 - - def readlink(self): - path = str(self.parent.resolve() / self.name) - if path in self._symlinks: - return self.with_segments(self._symlinks[path][0]) - elif path in self._files or path in self._directories: - raise OSError(errno.EINVAL, "Not a symlink", path) - else: - raise FileNotFoundError(errno.ENOENT, "File not found", path) - - def symlink_to(self, target, target_is_directory=False): - path = str(self.parent.resolve() / self.name) - parent = str(self.parent.resolve()) - if path in self._symlinks: - raise FileExistsError(errno.EEXIST, "File exists", path) - self._directories[parent].add(self.name) - self._symlinks[path] = str(target), target_is_directory - - -class DummyPathWithSymlinksTest(DummyPathTest): - cls = DummyPathWithSymlinks - can_symlink = True - - -class DummyPathWithSymlinksWalkTest(DummyPathWalkTest): - cls = DummyPathWithSymlinks - can_symlink = True if __name__ == "__main__": diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index e5f9848319021a..48a4c568651879 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -2931,6 +2931,22 @@ def test_pdb_issue_gh_108976(): (Pdb) continue """ +def test_pdb_issue_gh_127321(): + """See GH-127321 + breakpoint() should stop at a opcode that has a line number + >>> def test_function(): + ... import pdb; pdb_instance = pdb.Pdb(nosigint=True, readrc=False) + ... [1, 2] and pdb_instance.set_trace() + ... a = 1 + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'continue' + ... ]): + ... test_function() + > (4)test_function() + -> a = 1 + (Pdb) continue + """ + def test_pdb_issue_gh_80731(): """See GH-80731 diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index 265dc497ccb86c..a178d3353eecdf 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -443,6 +443,43 @@ def test_persid(self): highest protocol among opcodes = 0 ''') + def test_constants(self): + self.check_dis(b"(NI00\nI01\n\x89\x88t.", '''\ + 0: ( MARK + 1: N NONE + 2: I INT False + 6: I INT True + 10: \\x89 NEWFALSE + 11: \\x88 NEWTRUE + 12: t TUPLE (MARK at 0) + 13: . STOP +highest protocol among opcodes = 2 +''') + + def test_integers(self): + self.check_dis(b"(I0\nI1\nI10\nI011\nL12\nL13L\nL014\nL015L\nt.", '''\ + 0: ( MARK + 1: I INT 0 + 4: I INT 1 + 7: I INT 10 + 11: I INT 11 + 16: L LONG 12 + 20: L LONG 13 + 25: L LONG 14 + 30: L LONG 15 + 36: t TUPLE (MARK at 0) + 37: . STOP +highest protocol among opcodes = 0 +''') + + def test_nondecimal_integers(self): + self.check_dis_error(b'I0b10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0o10\n.', '', 'invalid literal for int') + self.check_dis_error(b'I0x10\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0b10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0o10L\n.', '', 'invalid literal for int') + self.check_dis_error(b'L0x10L\n.', '', 'invalid literal for int') + class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index 50e77cbbb6be13..8b8772c66ee654 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -114,6 +114,14 @@ def test_write_read_append(self): # write_history_file can create the target readline.write_history_file(hfilename) + # Negative values should be disallowed + with self.assertRaises(ValueError): + readline.append_history_file(-42, hfilename) + + # See gh-122431, using the minimum signed integer value caused a segfault + with self.assertRaises(ValueError): + readline.append_history_file(-2147483648, hfilename) + def test_nonascii_history(self): readline.clear_history() try: diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index d4f4a69a7a38c1..ab46ccbf004a3a 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -4,6 +4,7 @@ Note: test_regrtest cannot be run twice in parallel. """ +import _colorize import contextlib import dataclasses import glob @@ -21,6 +22,7 @@ import tempfile import textwrap import unittest +import unittest.mock from xml.etree import ElementTree from test import support @@ -1138,7 +1140,7 @@ def test_coverage(self): output = self.run_tests("--coverage", test) self.check_executed_tests(output, [test], stats=1) regex = (r'lines +cov% +module +\(path\)\n' - r'(?: *[0-9]+ *[0-9]{1,2}% *[^ ]+ +\([^)]+\)+)+') + r'(?: *[0-9]+ *[0-9]{1,2}\.[0-9]% *[^ ]+ +\([^)]+\)+)+') self.check_line(output, regex) def test_wait(self): @@ -2487,5 +2489,49 @@ def test_sanitize_xml(self): 'valid t\xe9xt \u20ac') +from test.libregrtest.results import TestResults + + +class TestColorized(unittest.TestCase): + def test_test_result_get_state(self): + # Arrange + green = _colorize.ANSIColors.GREEN + red = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + yellow = _colorize.ANSIColors.YELLOW + + good_results = TestResults() + good_results.good = ["good1", "good2"] + bad_results = TestResults() + bad_results.bad = ["bad1", "bad2"] + no_results = TestResults() + no_results.bad = [] + interrupted_results = TestResults() + interrupted_results.interrupted = True + interrupted_worker_bug = TestResults() + interrupted_worker_bug.interrupted = True + interrupted_worker_bug.worker_bug = True + + for results, expected in ( + (good_results, f"{green}SUCCESS{reset}"), + (bad_results, f"{red}FAILURE{reset}"), + (no_results, f"{yellow}NO TESTS RAN{reset}"), + (interrupted_results, f"{yellow}INTERRUPTED{reset}"), + ( + interrupted_worker_bug, + f"{yellow}INTERRUPTED{reset}, {red}WORKER BUG{reset}", + ), + ): + with self.subTest(results=results, expected=expected): + # Act + with unittest.mock.patch( + "_colorize.can_colorize", return_value=True + ): + result = results.get_state(fail_env_changed=False) + + # Assert + self.assertEqual(result, expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 7b3914f30e5f52..307d6e886c617f 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -5136,7 +5136,10 @@ def testLargeTimeout(self): # gh-126876: Check that a timeout larger than INT_MAX is replaced with # INT_MAX in the poll() code path. The following assertion must not # fail: assert(INT_MIN <= ms && ms <= INT_MAX). - large_timeout = _testcapi.INT_MAX + 1 + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 # test recv() with large timeout conn, addr = self.serv.accept() @@ -5151,7 +5154,10 @@ def testLargeTimeout(self): def _testLargeTimeout(self): # test sendall() with large timeout - large_timeout = _testcapi.INT_MAX + 1 + if _testcapi is not None: + large_timeout = _testcapi.INT_MAX + 1 + else: + large_timeout = 2147483648 self.cli.connect((HOST, self.port)) try: self.cli.settimeout(large_timeout) diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index db4e13222da9da..50cced3891d13a 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -433,6 +433,7 @@ def test_table_lock_cursor_dealloc(self): con.commit() cur = con.execute("select t from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() @@ -448,6 +449,7 @@ def dup(v): con.create_function("dup", 1, dup) cur = con.execute("select dup(t) from t") del cur + support.gc_collect() con.execute("drop table t") con.commit() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 9a3cf140d81241..635ae03a404988 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -578,7 +578,7 @@ def test_print_warning(self): 'Warning -- a\nWarning -- b\n') def test_has_strftime_extensions(self): - if support.is_emscripten or sys.platform == "win32": + if sys.platform == "win32": self.assertFalse(support.has_strftime_extensions) else: self.assertTrue(support.has_strftime_extensions) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 0df1a67ea2b720..ce504dc21af85f 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -53,6 +53,8 @@ def setUp(self): os.uname = self._get_uname # saving the environment self.name = os.name + self.prefix = sys.prefix + self.exec_prefix = sys.exec_prefix self.platform = sys.platform self.version = sys.version self._framework = sys._framework @@ -77,6 +79,8 @@ def tearDown(self): else: del os.uname os.name = self.name + sys.prefix = self.prefix + sys.exec_prefix = self.exec_prefix sys.platform = self.platform sys.version = self.version sys._framework = self._framework @@ -653,6 +657,27 @@ def test_sysconfigdata_json(self): self.assertEqual(system_config_vars, json_config_vars) + def test_sysconfig_config_vars_no_prefix_cache(self): + sys.prefix = 'prefix-AAA' + sys.exec_prefix = 'exec-prefix-AAA' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + + sys.prefix = 'prefix-BBB' + sys.exec_prefix = 'exec-prefix-BBB' + + config_vars = sysconfig.get_config_vars() + + self.assertEqual(config_vars['prefix'], sys.prefix) + self.assertEqual(config_vars['base'], sys.prefix) + self.assertEqual(config_vars['exec_prefix'], sys.exec_prefix) + self.assertEqual(config_vars['platbase'], sys.exec_prefix) + class MakefileTests(unittest.TestCase): diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index b666533466e578..3e164a12581dd1 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -1192,11 +1192,12 @@ def f(): resource.setrlimit(resource.RLIMIT_NPROC, (0, hard)) try: - _thread.start_new_thread(f, ()) + handle = _thread.start_joinable_thread(f) except RuntimeError: print('ok') else: print('!skip!') + handle.join() """ _, out, err = assert_python_ok("-u", "-c", code) out = out.strip() @@ -2103,6 +2104,85 @@ def test__all__(self): support.check__all__(self, threading, ('threading', '_thread'), extra=extra, not_exported=not_exported) + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_set_name(self): + # set_name() limit in bytes + truncate = getattr(_thread, "_NAME_MAXLEN", None) + limit = truncate or 100 + + tests = [ + # test short ASCII name + "CustomName", + + # test short non-ASCII name + "namé€", + + # embedded null character: name is truncated + # at the first null character + "embed\0null", + + # Test long ASCII names (not truncated) + "x" * limit, + + # Test long ASCII names (truncated) + "x" * (limit + 10), + + # Test long non-ASCII name (truncated) + "x" * (limit - 1) + "é€", + ] + if os_helper.FS_NONASCII: + tests.append(f"nonascii:{os_helper.FS_NONASCII}") + if os_helper.TESTFN_UNENCODABLE: + tests.append(os_helper.TESTFN_UNENCODABLE) + + if sys.platform.startswith("solaris"): + encoding = "utf-8" + else: + encoding = sys.getfilesystemencoding() + + def work(): + nonlocal work_name + work_name = _thread._get_name() + + for name in tests: + encoded = name.encode(encoding, "replace") + if b'\0' in encoded: + encoded = encoded.split(b'\0', 1)[0] + if truncate is not None: + encoded = encoded[:truncate] + if sys.platform.startswith("solaris"): + expected = encoded.decode("utf-8", "surrogateescape") + else: + expected = os.fsdecode(encoded) + + with self.subTest(name=name, expected=expected): + work_name = None + thread = threading.Thread(target=work, name=name) + thread.start() + thread.join() + self.assertEqual(work_name, expected, + f"{len(work_name)=} and {len(expected)=}") + + @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_change_name(self): + # Change the name of a thread while the thread is running + + name1 = None + name2 = None + def work(): + nonlocal name1, name2 + name1 = _thread._get_name() + threading.current_thread().name = "new name" + name2 = _thread._get_name() + + thread = threading.Thread(target=work, name="name") + thread.start() + thread.join() + self.assertEqual(name1, "name") + self.assertEqual(name2, "new name") + class InterruptMainTests(unittest.TestCase): def check_interrupt_main_with_signal_handler(self, signum): diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d368f08b610870..92398300f26577 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -116,6 +116,7 @@ def test_clock_monotonic(self): 'need time.pthread_getcpuclockid()') @unittest.skipUnless(hasattr(time, 'clock_gettime'), 'need time.clock_gettime()') + @unittest.skipIf(support.is_emscripten, "Fails to find clock") def test_pthread_getcpuclockid(self): clk_id = time.pthread_getcpuclockid(threading.get_ident()) self.assertTrue(type(clk_id) is int) @@ -539,6 +540,9 @@ def test_perf_counter(self): @unittest.skipIf( support.is_wasi, "process_time not available on WASI" ) + @unittest.skipIf( + support.is_emscripten, "process_time present but doesn't exclude sleep" + ) def test_process_time(self): # process_time() should not include time spend during a sleep start = time.process_time() diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 579ce2af9fa0bf..475edcbd5338a7 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -123,9 +123,9 @@ def test_tk_setPalette(self): def test_after(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Without function, sleeps for ms. self.assertIsNone(root.after(1)) @@ -161,12 +161,18 @@ def callback(start=0, step=1): root.update() # Process all pending events. self.assertEqual(count, 53) + # Set up with callback with keyword args. + count = 0 + timer1 = root.after(0, callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_idle(self): root = self.root - def callback(start=0, step=1): + def callback(start=0, step=1, *, end=0): nonlocal count - count = start + step + count = start + step + end # Set up with callback with no args. count = 0 @@ -193,6 +199,12 @@ def callback(start=0, step=1): with self.assertRaises(tkinter.TclError): root.tk.call(script) + # Set up with callback with keyword args. + count = 0 + idle1 = root.after_idle(callback, 42, step=11, end=1) + root.update() # Process all pending events. + self.assertEqual(count, 54) + def test_after_cancel(self): root = self.root diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 93966ee31d0a01..e7e42531916d0d 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -412,7 +412,7 @@ def test_issue9936(self): coverage = {} for line in stdout: lines, cov, module = line.split()[:3] - coverage[module] = (int(lines), int(cov[:-1])) + coverage[module] = (float(lines), float(cov[:-1])) # XXX This is needed to run regrtest.py as a script modname = trace._fullmodname(sys.modules[modname].__file__) self.assertIn(modname, coverage) @@ -553,7 +553,7 @@ def f(): stdout = stdout.decode() self.assertEqual(status, 0) self.assertIn('lines cov% module (path)', stdout) - self.assertIn(f'6 100% {modulename} ({filename})', stdout) + self.assertIn(f'6 100.0% {modulename} ({filename})', stdout) def test_run_as_module(self): assert_python_ok('-m', 'trace', '-l', '--module', 'timeit', '-n', '1') diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index ec69412f5511eb..31f0a61d6a9d59 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -2097,6 +2097,7 @@ def deep_eg(self): return e @cpython_only + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_capi(self): from _testcapi import exception_print LIMIT = 75 @@ -2108,6 +2109,7 @@ def test_exception_group_deep_recursion_capi(self): self.assertIn('ExceptionGroup', output) self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + @support.skip_emscripten_stack_overflow() def test_exception_group_deep_recursion_traceback(self): LIMIT = 75 eg = self.deep_eg() @@ -4488,9 +4490,8 @@ class MiscTest(unittest.TestCase): def test_all(self): expected = set() - denylist = {'print_list'} for name in dir(traceback): - if name.startswith('_') or name in denylist: + if name.startswith('_'): continue module_object = getattr(traceback, name) if getattr(module_object, '__module__', None) == 'traceback': diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 66abe73f8d766d..e109a65741309a 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -2,7 +2,7 @@ import unittest import dis from test import support -from test.support import import_helper, requires_specialization +from test.support import import_helper, requires_specialization, requires_specialization_ft try: from sys import _clear_type_cache except ImportError: @@ -110,7 +110,6 @@ class HolderSub(Holder): HolderSub.value @support.cpython_only -@requires_specialization class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): _clear_type_cache() @@ -140,6 +139,7 @@ def _check_specialization(self, func, arg, opname, *, should_specialize): else: self.assertIn(opname, self._all_opnames(func)) + @requires_specialization def test_class_load_attr_specialization_user_type(self): class A: def foo(self): @@ -160,6 +160,7 @@ def load_foo_2(type_): self._check_specialization(load_foo_2, A, "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_class_load_attr_specialization_static_type(self): self.assertNotEqual(type_get_version(str), 0) self.assertNotEqual(type_get_version(bytes), 0) @@ -171,6 +172,7 @@ def get_capitalize_1(type_): self.assertEqual(get_capitalize_1(str)('hello'), 'Hello') self.assertEqual(get_capitalize_1(bytes)(b'hello'), b'Hello') + @requires_specialization def test_property_load_attr_specialization_user_type(self): class G: @property @@ -192,6 +194,7 @@ def load_x_2(instance): self._check_specialization(load_x_2, G(), "LOAD_ATTR", should_specialize=False) + @requires_specialization def test_store_attr_specialization_user_type(self): class B: __slots__ = ("bar",) @@ -211,6 +214,7 @@ def store_bar_2(type_): self._check_specialization(store_bar_2, B(), "STORE_ATTR", should_specialize=False) + @requires_specialization_ft def test_class_call_specialization_user_type(self): class F: def __init__(self): @@ -231,6 +235,7 @@ def call_class_2(type_): self._check_specialization(call_class_2, F, "CALL", should_specialize=False) + @requires_specialization def test_to_bool_specialization_user_type(self): class H: pass diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 00ef55bdf9bc83..8ea244bff05c5f 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -2,6 +2,7 @@ import contextvars import unittest from test import support +from test.support import force_not_colorized support.requires_working_socket(module=True) @@ -252,6 +253,7 @@ async def on_cleanup(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup']) + @force_not_colorized def test_exception_in_tear_clean_up(self): class Test(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 7241cf59f73d4f..0b46f338ac77e1 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -4,6 +4,7 @@ from test import support import unittest import test.test_unittest +from test.support import force_not_colorized from test.test_unittest.test_result import BufferedWriter @@ -120,6 +121,7 @@ def run(self, test): self.assertEqual(['test.test_unittest', 'test.test_unittest2'], program.testNames) + @force_not_colorized def test_NonExit(self): stream = BufferedWriter() program = unittest.main(exit=False, @@ -135,6 +137,7 @@ def test_NonExit(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_Exit(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -152,6 +155,7 @@ def test_Exit(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitAsDefault(self): stream = BufferedWriter() with self.assertRaises(SystemExit): @@ -167,6 +171,7 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitSkippedSuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: @@ -179,6 +184,7 @@ def test_ExitSkippedSuite(self): expected = '\n\nOK (skipped=1)\n' self.assertTrue(out.endswith(expected)) + @force_not_colorized def test_ExitEmptySuite(self): stream = BufferedWriter() with self.assertRaises(SystemExit) as cm: diff --git a/Lib/test/test_unittest/test_result.py b/Lib/test/test_unittest/test_result.py index 4e5ec54e9c892a..746b9fa2677717 100644 --- a/Lib/test/test_unittest/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -7,6 +7,7 @@ import traceback import unittest from unittest.util import strclass +from test.support import force_not_colorized from test.test_unittest.support import BufferedWriter @@ -14,7 +15,7 @@ class MockTraceback(object): class TracebackException: def __init__(self, *args, **kwargs): self.capture_locals = kwargs.get('capture_locals', False) - def format(self): + def format(self, **kwargs): result = ['A traceback'] if self.capture_locals: result.append('locals') @@ -205,6 +206,7 @@ def test_1(self): self.assertIs(test_case, test) self.assertIsInstance(formatted_exc, str) + @force_not_colorized def test_addFailure_filter_traceback_frames(self): class Foo(unittest.TestCase): def test_1(self): @@ -231,6 +233,7 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + @force_not_colorized def test_addFailure_filter_traceback_frames_context(self): class Foo(unittest.TestCase): def test_1(self): @@ -260,6 +263,7 @@ def get_exc_info(): self.assertEqual(len(dropped), 1) self.assertIn("raise self.failureException(msg)", dropped[0]) + @force_not_colorized def test_addFailure_filter_traceback_frames_chained_exception_self_loop(self): class Foo(unittest.TestCase): def test_1(self): @@ -285,6 +289,7 @@ def get_exc_info(): formatted_exc = result.failures[0][1] self.assertEqual(formatted_exc.count("Exception: Loop\n"), 1) + @force_not_colorized def test_addFailure_filter_traceback_frames_chained_exception_cycle(self): class Foo(unittest.TestCase): def test_1(self): @@ -446,6 +451,7 @@ def testFailFast(self): result.addUnexpectedSuccess(None) self.assertTrue(result.shouldStop) + @force_not_colorized def testFailFastSetByRunner(self): stream = BufferedWriter() runner = unittest.TextTestRunner(stream=stream, failfast=True) @@ -619,6 +625,7 @@ def _run_test(self, test_name, verbosity, tearDownError=None): test.run(result) return stream.getvalue() + @force_not_colorized def testDotsOutput(self): self.assertEqual(self._run_test('testSuccess', 1), '.') self.assertEqual(self._run_test('testSkip', 1), 's') @@ -627,6 +634,7 @@ def testDotsOutput(self): self.assertEqual(self._run_test('testExpectedFailure', 1), 'x') self.assertEqual(self._run_test('testUnexpectedSuccess', 1), 'u') + @force_not_colorized def testLongOutput(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSuccess', 2), @@ -642,17 +650,21 @@ def testLongOutput(self): self.assertEqual(self._run_test('testUnexpectedSuccess', 2), f'testUnexpectedSuccess ({classname}.testUnexpectedSuccess) ... unexpected success\n') + @force_not_colorized def testDotsOutputSubTestSuccess(self): self.assertEqual(self._run_test('testSubTestSuccess', 1), '.') + @force_not_colorized def testLongOutputSubTestSuccess(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSubTestSuccess', 2), f'testSubTestSuccess ({classname}.testSubTestSuccess) ... ok\n') + @force_not_colorized def testDotsOutputSubTestMixed(self): self.assertEqual(self._run_test('testSubTestMixed', 1), 'sFE') + @force_not_colorized def testLongOutputSubTestMixed(self): classname = f'{__name__}.{self.Test.__qualname__}' self.assertEqual(self._run_test('testSubTestMixed', 2), @@ -661,6 +673,7 @@ def testLongOutputSubTestMixed(self): f' testSubTestMixed ({classname}.testSubTestMixed) [fail] (c=3) ... FAIL\n' f' testSubTestMixed ({classname}.testSubTestMixed) [error] (d=4) ... ERROR\n') + @force_not_colorized def testDotsOutputTearDownFail(self): out = self._run_test('testSuccess', 1, AssertionError('fail')) self.assertEqual(out, 'F') @@ -671,6 +684,7 @@ def testDotsOutputTearDownFail(self): out = self._run_test('testSkip', 1, AssertionError('fail')) self.assertEqual(out, 'sF') + @force_not_colorized def testLongOutputTearDownFail(self): classname = f'{__name__}.{self.Test.__qualname__}' out = self._run_test('testSuccess', 2, AssertionError('fail')) diff --git a/Lib/test/test_unittest/test_runner.py b/Lib/test/test_unittest/test_runner.py index 1b9cef43e3f9c5..1131cd73128866 100644 --- a/Lib/test/test_unittest/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -4,6 +4,7 @@ import pickle import subprocess from test import support +from test.support import force_not_colorized import unittest from unittest.case import _Outcome @@ -106,6 +107,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -416,6 +418,7 @@ def cleanup2(): self.assertIsInstance(e2[1], CustomError) self.assertEqual(str(e2[1]), 'cleanup1') + @force_not_colorized def test_with_errors_addCleanUp(self): ordering = [] class TestableTest(unittest.TestCase): @@ -439,6 +442,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) + @force_not_colorized def test_run_with_errors_addClassCleanUp(self): ordering = [] class TestableTest(unittest.TestCase): @@ -462,6 +466,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) + @force_not_colorized def test_with_errors_in_addClassCleanup_and_setUps(self): ordering = [] class_blow_up = False @@ -514,6 +519,7 @@ def tearDownClass(cls): ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) + @force_not_colorized def test_with_errors_in_tearDownClass(self): ordering = [] class TestableTest(unittest.TestCase): @@ -590,6 +596,7 @@ def test(self): 'inner setup', 'inner test', 'inner cleanup', 'end outer test', 'outer cleanup']) + @force_not_colorized def test_run_empty_suite_error_message(self): class EmptyTest(unittest.TestCase): pass @@ -663,6 +670,7 @@ class Module(object): self.assertEqual(cleanups, [((1, 2), {'function': 'hello'})]) + @force_not_colorized def test_run_module_cleanUp(self): blowUp = True ordering = [] @@ -802,6 +810,7 @@ def tearDownClass(cls): 'tearDownClass', 'cleanup_good']) self.assertEqual(unittest.case._module_cleanups, []) + @force_not_colorized def test_run_module_cleanUp_when_teardown_exception(self): ordering = [] class Module(object): @@ -963,6 +972,7 @@ def testNothing(self): self.assertEqual(cleanups, [((1, 2), {'function': 3, 'self': 4})]) + @force_not_colorized def test_with_errors_in_addClassCleanup(self): ordering = [] @@ -996,6 +1006,7 @@ def tearDownClass(cls): ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) + @force_not_colorized def test_with_errors_in_addCleanup(self): ordering = [] class Module(object): @@ -1026,6 +1037,7 @@ def tearDown(self): ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) + @force_not_colorized def test_with_errors_in_addModuleCleanup_and_setUps(self): ordering = [] module_blow_up = False @@ -1318,6 +1330,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @force_not_colorized @support.requires_subprocess() def test_warnings(self): """ diff --git a/Lib/test/test_unittest/test_skipping.py b/Lib/test/test_unittest/test_skipping.py index f146dcac18ecc0..f5cb860c60b156 100644 --- a/Lib/test/test_unittest/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,5 +1,6 @@ import unittest +from test.support import force_not_colorized from test.test_unittest.support import LoggingResult @@ -293,6 +294,7 @@ def test_die(self): self.assertFalse(result.unexpectedSuccesses) self.assertTrue(result.wasSuccessful()) + @force_not_colorized def test_expected_failure_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure @@ -372,6 +374,7 @@ def test_die(self): self.assertEqual(result.unexpectedSuccesses, [test]) self.assertFalse(result.wasSuccessful()) + @force_not_colorized def test_unexpected_success_and_fail_in_cleanup(self): class Foo(unittest.TestCase): @unittest.expectedFailure diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 00e46990c406ac..042d3b35b77022 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -156,7 +156,7 @@ def test_headers(self): self.assertIsInstance(self.returned_obj.headers, email.message.Message) def test_url(self): - self.assertEqual(self.returned_obj.url, "file://" + self.quoted_pathname) + self.assertEqual(self.returned_obj.url, "file:" + self.quoted_pathname) def test_status(self): self.assertIsNone(self.returned_obj.status) @@ -165,7 +165,7 @@ def test_info(self): self.assertIsInstance(self.returned_obj.info(), email.message.Message) def test_geturl(self): - self.assertEqual(self.returned_obj.geturl(), "file://" + self.quoted_pathname) + self.assertEqual(self.returned_obj.geturl(), "file:" + self.quoted_pathname) def test_getcode(self): self.assertIsNone(self.returned_obj.getcode()) @@ -471,11 +471,14 @@ def test_missing_localfile(self): def test_file_notexists(self): fd, tmp_file = tempfile.mkstemp() - tmp_fileurl = 'file://localhost/' + tmp_file.replace(os.path.sep, '/') + tmp_file_canon_url = 'file:' + urllib.request.pathname2url(tmp_file) + parsed = urllib.parse.urlsplit(tmp_file_canon_url) + tmp_fileurl = parsed._replace(netloc='localhost').geturl() try: self.assertTrue(os.path.exists(tmp_file)) with urllib.request.urlopen(tmp_fileurl) as fobj: self.assertTrue(fobj) + self.assertEqual(fobj.url, tmp_file_canon_url) finally: os.close(fd) os.unlink(tmp_file) @@ -609,7 +612,7 @@ def tearDown(self): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) - return "file://%s" % urllib.request.pathname2url(filePath) + return "file:" + urllib.request.pathname2url(filePath) def createNewTempFile(self, data=b""): """Creates a new temporary file containing the specified data, diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 99ad11cf0552eb..4a9e653515be5b 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -23,7 +23,7 @@ _proxy_bypass_winreg_override, _proxy_bypass_macosx_sysconf, AbstractDigestAuthHandler) -from urllib.parse import urlparse +from urllib.parse import urlsplit import urllib.error import http.client @@ -717,14 +717,6 @@ def test_processors(self): self.assertIsInstance(args[1], MockResponse) -def sanepathname2url(path): - urlpath = urllib.request.pathname2url(path) - if os.name == "nt" and urlpath.startswith("///"): - urlpath = urlpath[2:] - # XXX don't ask me about the mac... - return urlpath - - class HandlerTests(unittest.TestCase): def test_ftp(self): @@ -818,19 +810,22 @@ def test_file(self): o = h.parent = MockOpener() TESTFN = os_helper.TESTFN - urlpath = sanepathname2url(os.path.abspath(TESTFN)) towrite = b"hello, world\n" + canonurl = 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)) + parsed = urlsplit(canonurl) + if parsed.netloc: + raise unittest.SkipTest("non-local working directory") urls = [ - "file://localhost%s" % urlpath, - "file://%s" % urlpath, - "file://%s%s" % (socket.gethostbyname('localhost'), urlpath), + canonurl, + parsed._replace(netloc='localhost').geturl(), + parsed._replace(netloc=socket.gethostbyname('localhost')).geturl(), ] try: localaddr = socket.gethostbyname(socket.gethostname()) except socket.gaierror: localaddr = '' if localaddr: - urls.append("file://%s%s" % (localaddr, urlpath)) + urls.append(parsed._replace(netloc=localaddr).geturl()) for url in urls: f = open(TESTFN, "wb") @@ -855,10 +850,10 @@ def test_file(self): self.assertEqual(headers["Content-type"], "text/plain") self.assertEqual(headers["Content-length"], "13") self.assertEqual(headers["Last-modified"], modified) - self.assertEqual(respurl, url) + self.assertEqual(respurl, canonurl) for url in [ - "file://localhost:80%s" % urlpath, + parsed._replace(netloc='localhost:80').geturl(), "file:///file_does_not_exist.txt", "file://not-a-local-host.com//dir/file.txt", "file://%s:80%s/%s" % (socket.gethostbyname('localhost'), @@ -1156,13 +1151,13 @@ def test_full_url_setter(self): r = Request('http://example.com') for url in urls: r.full_url = url - parsed = urlparse(url) + parsed = urlsplit(url) self.assertEqual(r.get_full_url(), url) # full_url setter uses splittag to split into components. # splittag sets the fragment as None while urlparse sets it to '' self.assertEqual(r.fragment or '', parsed.fragment) - self.assertEqual(urlparse(r.get_full_url()).query, parsed.query) + self.assertEqual(urlsplit(r.get_full_url()).query, parsed.query) def test_full_url_deleter(self): r = Request('http://www.example.com') diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index f0874d8d3ce463..b84290a7368c29 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -4,7 +4,6 @@ from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied -from test.test_urllib2 import sanepathname2url import os import socket @@ -151,7 +150,7 @@ def test_file(self): f.write('hi there\n') f.close() urls = [ - 'file:' + sanepathname2url(os.path.abspath(TESTFN)), + 'file:' + urllib.request.pathname2url(os.path.abspath(TESTFN)), ('file:///nonsensename/etc/passwd', None, urllib.error.URLError), ] diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 3a0fc572f457ff..db19af419bdeab 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -57,6 +57,7 @@ def test_del_attribute(self): del element.attrib self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) + @unittest.skipIf(support.is_emscripten, "segfaults") def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/Lib/threading.py b/Lib/threading.py index 94ea2f08178369..78e591124278fc 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -48,6 +48,10 @@ __all__.append('get_native_id') except AttributeError: _HAVE_THREAD_NATIVE_ID = False +try: + _set_name = _thread.set_name +except AttributeError: + _set_name = None ThreadError = _thread.error try: _CRLock = _thread.RLock @@ -1022,11 +1026,20 @@ def _set_ident(self): def _set_native_id(self): self._native_id = get_native_id() + def _set_os_name(self): + if _set_name is None or not self._name: + return + try: + _set_name(self._name) + except OSError: + pass + def _bootstrap_inner(self): try: self._set_ident() if _HAVE_THREAD_NATIVE_ID: self._set_native_id() + self._set_os_name() self._started.set() with _active_limbo_lock: _active[self._ident] = self @@ -1106,6 +1119,8 @@ def name(self): def name(self, name): assert self._initialized, "Thread.__init__() not called" self._name = str(name) + if get_ident() == self._ident: + self._set_os_name() @property def ident(self): diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index dd7b3e138f4236..bfec04bb6c1e6e 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -847,7 +847,7 @@ def tk_focusPrev(self): if not name: return None return self._nametowidget(name) - def after(self, ms, func=None, *args): + def after(self, ms, func=None, *args, **kw): """Call function once after given time. MS specifies the time in milliseconds. FUNC gives the @@ -861,7 +861,7 @@ def after(self, ms, func=None, *args): else: def callit(): try: - func(*args) + func(*args, **kw) finally: try: self.deletecommand(name) @@ -875,13 +875,13 @@ def callit(): name = self._register(callit) return self.tk.call('after', ms, name) - def after_idle(self, func, *args): + def after_idle(self, func, *args, **kw): """Call FUNC once if the Tcl main loop has no event to process. Return an identifier to cancel the scheduling with after_cancel.""" - return self.after('idle', func, *args) + return self.after('idle', func, *args, **kw) def after_cancel(self, id): """Cancel scheduling of function identified with ID. diff --git a/Lib/token.py b/Lib/token.py index b620317106e173..54d7cdccadc79a 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -1,7 +1,8 @@ """Token constants.""" # Auto-generated by Tools/build/generate_token.py -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] ENDMARKER = 0 NAME = 1 diff --git a/Lib/trace.py b/Lib/trace.py index bb3d34fd8d6550..a87bc6d61a884f 100644 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -279,14 +279,13 @@ def write_results(self, show_missing=True, summary=False, coverdir=None, *, n_hits, n_lines = self.write_results_file(coverpath, source, lnotab, count, encoding) if summary and n_lines: - percent = int(100 * n_hits / n_lines) - sums[modulename] = n_lines, percent, modulename, filename + sums[modulename] = n_lines, n_hits, modulename, filename if summary and sums: print("lines cov% module (path)") for m in sorted(sums): - n_lines, percent, modulename, filename = sums[m] - print("%5d %3d%% %s (%s)" % sums[m]) + n_lines, n_hits, modulename, filename = sums[m] + print(f"{n_lines:5d} {n_hits/n_lines:.1%} {modulename} ({filename})") if self.outfile: # try and store counts and module info into self.outfile diff --git a/Lib/traceback.py b/Lib/traceback.py index f73149271b9bc9..6367c00e4d4b86 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -15,7 +15,7 @@ 'format_tb', 'print_exc', 'format_exc', 'print_exception', 'print_last', 'print_stack', 'print_tb', 'clear_frames', 'FrameSummary', 'StackSummary', 'TracebackException', - 'walk_stack', 'walk_tb'] + 'walk_stack', 'walk_tb', 'print_list'] # # Formatting and printing lists of traceback lines. diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 3ace0a5b7bf2ef..97262735aa8311 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -189,7 +189,9 @@ def _exc_info_to_string(self, err, test): tb_e = traceback.TracebackException( exctype, value, tb, capture_locals=self.tb_locals, compact=True) - msgLines = list(tb_e.format()) + from _colorize import can_colorize + + msgLines = list(tb_e.format(colorize=can_colorize())) if self.buffer: output = sys.stdout.getvalue() diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index 2bcadf0c998bd9..d60c295a1eddf7 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -4,6 +4,8 @@ import time import warnings +from _colorize import get_colors + from . import result from .case import _SubTest from .signals import registerResult @@ -13,18 +15,18 @@ class _WritelnDecorator(object): """Used to decorate file-like objects with a handy 'writeln' method""" - def __init__(self,stream): + def __init__(self, stream): self.stream = stream def __getattr__(self, attr): if attr in ('stream', '__getstate__'): raise AttributeError(attr) - return getattr(self.stream,attr) + return getattr(self.stream, attr) def writeln(self, arg=None): if arg: self.write(arg) - self.write('\n') # text-mode streams translate to \r\n if needed + self.write('\n') # text-mode streams translate to \r\n if needed class TextTestResult(result.TestResult): @@ -43,6 +45,7 @@ def __init__(self, stream, descriptions, verbosity, *, durations=None): self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions + self._ansi = get_colors() self._newline = True self.durations = durations @@ -76,86 +79,102 @@ def _write_status(self, test, status): def addSubTest(self, test, subtest, err): if err is not None: + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: if issubclass(err[0], subtest.failureException): - self._write_status(subtest, "FAIL") + self._write_status(subtest, f"{red}FAIL{reset}") else: - self._write_status(subtest, "ERROR") + self._write_status(subtest, f"{red}ERROR{reset}") elif self.dots: if issubclass(err[0], subtest.failureException): - self.stream.write('F') + self.stream.write(f"{red}F{reset}") else: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() super(TextTestResult, self).addSubTest(test, subtest, err) def addSuccess(self, test): super(TextTestResult, self).addSuccess(test) + green, reset = self._ansi.GREEN, self._ansi.RESET if self.showAll: - self._write_status(test, "ok") + self._write_status(test, f"{green}ok{reset}") elif self.dots: - self.stream.write('.') + self.stream.write(f"{green}.{reset}") self.stream.flush() def addError(self, test, err): super(TextTestResult, self).addError(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "ERROR") + self._write_status(test, f"{red}ERROR{reset}") elif self.dots: - self.stream.write('E') + self.stream.write(f"{red}E{reset}") self.stream.flush() def addFailure(self, test, err): super(TextTestResult, self).addFailure(test, err) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self._write_status(test, "FAIL") + self._write_status(test, f"{red}FAIL{reset}") elif self.dots: - self.stream.write('F') + self.stream.write(f"{red}F{reset}") self.stream.flush() def addSkip(self, test, reason): super(TextTestResult, self).addSkip(test, reason) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self._write_status(test, "skipped {0!r}".format(reason)) + self._write_status(test, f"{yellow}skipped{reset} {reason!r}") elif self.dots: - self.stream.write("s") + self.stream.write(f"{yellow}s{reset}") self.stream.flush() def addExpectedFailure(self, test, err): super(TextTestResult, self).addExpectedFailure(test, err) + yellow, reset = self._ansi.YELLOW, self._ansi.RESET if self.showAll: - self.stream.writeln("expected failure") + self.stream.writeln(f"{yellow}expected failure{reset}") self.stream.flush() elif self.dots: - self.stream.write("x") + self.stream.write(f"{yellow}x{reset}") self.stream.flush() def addUnexpectedSuccess(self, test): super(TextTestResult, self).addUnexpectedSuccess(test) + red, reset = self._ansi.RED, self._ansi.RESET if self.showAll: - self.stream.writeln("unexpected success") + self.stream.writeln(f"{red}unexpected success{reset}") self.stream.flush() elif self.dots: - self.stream.write("u") + self.stream.write(f"{red}u{reset}") self.stream.flush() def printErrors(self): + bold_red = self._ansi.BOLD_RED + red = self._ansi.RED + reset = self._ansi.RESET if self.dots or self.showAll: self.stream.writeln() self.stream.flush() - self.printErrorList('ERROR', self.errors) - self.printErrorList('FAIL', self.failures) - unexpectedSuccesses = getattr(self, 'unexpectedSuccesses', ()) + self.printErrorList(f"{red}ERROR{reset}", self.errors) + self.printErrorList(f"{red}FAIL{reset}", self.failures) + unexpectedSuccesses = getattr(self, "unexpectedSuccesses", ()) if unexpectedSuccesses: self.stream.writeln(self.separator1) for test in unexpectedSuccesses: - self.stream.writeln(f"UNEXPECTED SUCCESS: {self.getDescription(test)}") + self.stream.writeln( + f"{red}UNEXPECTED SUCCESS{bold_red}: " + f"{self.getDescription(test)}{reset}" + ) self.stream.flush() def printErrorList(self, flavour, errors): + bold_red, reset = self._ansi.BOLD_RED, self._ansi.RESET for test, err in errors: self.stream.writeln(self.separator1) - self.stream.writeln("%s: %s" % (flavour,self.getDescription(test))) + self.stream.writeln( + f"{flavour}{bold_red}: {self.getDescription(test)}{reset}" + ) self.stream.writeln(self.separator2) self.stream.writeln("%s" % err) self.stream.flush() @@ -232,7 +251,7 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - startTime = time.perf_counter() + start_time = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -242,8 +261,8 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.perf_counter() - timeTaken = stopTime - startTime + stop_time = time.perf_counter() + time_taken = stop_time - start_time result.printErrors() if self.durations is not None: self._printDurations(result) @@ -253,10 +272,10 @@ def run(self, test): run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % - (run, run != 1 and "s" or "", timeTaken)) + (run, run != 1 and "s" or "", time_taken)) self.stream.writeln() - expectedFails = unexpectedSuccesses = skipped = 0 + expected_fails = unexpected_successes = skipped = 0 try: results = map(len, (result.expectedFailures, result.unexpectedSuccesses, @@ -264,26 +283,35 @@ def run(self, test): except AttributeError: pass else: - expectedFails, unexpectedSuccesses, skipped = results + expected_fails, unexpected_successes, skipped = results infos = [] + ansi = get_colors() + bold_red = ansi.BOLD_RED + green = ansi.GREEN + red = ansi.RED + reset = ansi.RESET + yellow = ansi.YELLOW + if not result.wasSuccessful(): - self.stream.write("FAILED") + self.stream.write(f"{bold_red}FAILED{reset}") failed, errored = len(result.failures), len(result.errors) if failed: - infos.append("failures=%d" % failed) + infos.append(f"{bold_red}failures={failed}{reset}") if errored: - infos.append("errors=%d" % errored) + infos.append(f"{bold_red}errors={errored}{reset}") elif run == 0 and not skipped: - self.stream.write("NO TESTS RAN") + self.stream.write(f"{yellow}NO TESTS RAN{reset}") else: - self.stream.write("OK") + self.stream.write(f"{green}OK{reset}") if skipped: - infos.append("skipped=%d" % skipped) - if expectedFails: - infos.append("expected failures=%d" % expectedFails) - if unexpectedSuccesses: - infos.append("unexpected successes=%d" % unexpectedSuccesses) + infos.append(f"{yellow}skipped={skipped}{reset}") + if expected_fails: + infos.append(f"{yellow}expected failures={expected_fails}{reset}") + if unexpected_successes: + infos.append( + f"{red}unexpected successes={unexpected_successes}{reset}" + ) if infos: self.stream.writeln(" (%s)" % (", ".join(infos),)) else: diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 1fcaa89188188d..c5a6a18a32bba1 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -876,9 +876,9 @@ def find_user_password(self, realm, authuri): class HTTPPasswordMgrWithPriorAuth(HTTPPasswordMgrWithDefaultRealm): - def __init__(self, *args, **kwargs): + def __init__(self): self.authenticated = {} - super().__init__(*args, **kwargs) + super().__init__() def add_password(self, realm, uri, user, passwd, is_authenticated=False): self.update_authenticated(uri, is_authenticated) @@ -1488,10 +1488,7 @@ def open_local_file(self, req): host, port = _splitport(host) if not host or \ (not port and _safe_gethostbyname(host) in self.get_names()): - if host: - origurl = 'file://' + host + filename - else: - origurl = 'file://' + filename + origurl = 'file:' + pathname2url(localfile) return addinfourl(open(localfile, 'rb'), headers, origurl) except OSError as exp: raise URLError(exp, exp.filename) diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index ca1af84e6705fe..dc4c9ef3531991 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -103,8 +103,6 @@ def _venv_path(self, env_dir, name): vars = { 'base': env_dir, 'platbase': env_dir, - 'installed_base': env_dir, - 'installed_platbase': env_dir, } return sysconfig.get_path(name, scheme='venv', vars=vars) @@ -175,9 +173,20 @@ def create_if_needed(d): context.python_dir = dirname context.python_exe = exename binpath = self._venv_path(env_dir, 'scripts') - incpath = self._venv_path(env_dir, 'include') libpath = self._venv_path(env_dir, 'purelib') + # PEP 405 says venvs should create a local include directory. + # See https://peps.python.org/pep-0405/#include-files + # XXX: This directory is not exposed in sysconfig or anywhere else, and + # doesn't seem to be utilized by modern packaging tools. We keep it + # for backwards-compatibility, and to follow the PEP, but I would + # recommend against using it, as most tooling does not pass it to + # compilers. Instead, until we standardize a site-specific include + # directory, I would recommend installing headers as package data, + # and providing some sort of API to get the include directories. + # Example: https://numpy.org/doc/2.1/reference/generated/numpy.get_include.html + incpath = os.path.join(env_dir, 'Include' if os.name == 'nt' else 'include') + context.inc_path = incpath create_if_needed(incpath) context.lib_path = libpath diff --git a/Makefile.pre.in b/Makefile.pre.in index 724354746b8d81..7b66802147dc3a 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -269,10 +269,6 @@ SRCDIRS= @SRCDIRS@ # Other subdirectories SUBDIRSTOO= Include Lib Misc -# assets for Emscripten browser builds -WASM_ASSETS_DIR=.$(prefix) -WASM_STDLIB=$(WASM_ASSETS_DIR)/lib/python$(VERSION)/os.py - # Files and directories to be distributed CONFIGFILES= configure configure.ac acconfig.h pyconfig.h.in Makefile.pre.in DISTFILES= README.rst ChangeLog $(CONFIGFILES) @@ -737,6 +733,9 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ python-config checksharedmods +.PHONY: build_emscripten +build_emscripten: build_wasm web_example + # Check that the source is clean when building out of source. .PHONY: check-clean-src check-clean-src: @@ -1016,23 +1015,38 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) else true; \ fi -# wasm32-emscripten browser build -# wasm assets directory is relative to current build dir, e.g. "./usr/local". -# --preload-file turns a relative asset path into an absolute path. +# wasm32-emscripten browser web example + +WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ +web_example/python.html: $(WEBEX_DIR)/python.html + @mkdir -p web_example + @cp $< $@ + +web_example/python.worker.mjs: $(WEBEX_DIR)/python.worker.mjs + @mkdir -p web_example + @cp $< $@ + +web_example/server.py: $(WEBEX_DIR)/server.py + @mkdir -p web_example + @cp $< $@ -.PHONY: wasm_stdlib -wasm_stdlib: $(WASM_STDLIB) -$(WASM_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(srcdir)/Tools/wasm/wasm_assets.py \ +WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip +$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(WEBEX_DIR)/wasm_assets.py \ Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(srcdir)/Tools/wasm/wasm_assets.py \ - --buildroot . --prefix $(prefix) + $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ -python.html: $(srcdir)/Tools/wasm/python.html python.worker.js - @cp $(srcdir)/Tools/wasm/python.html $@ +web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + cp python.mjs web_example/python.mjs + cp python.wasm web_example/python.wasm -python.worker.js: $(srcdir)/Tools/wasm/python.worker.js - @cp $(srcdir)/Tools/wasm/python.worker.js $@ +.PHONY: web_example +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) ############################################################################ # Header files @@ -2132,7 +2146,6 @@ testuniversal: all # This must be run *after* a `make install` has completed the build. The # `--with-framework-name` argument *cannot* be used when configuring the build. XCFOLDER:=iOSTestbed.$(MULTIARCH).$(shell date +%s) -XCRESULT=$(XCFOLDER)/$(MULTIARCH).xcresult .PHONY: testios testios: @if test "$(MACHDEP)" != "ios"; then \ @@ -2151,29 +2164,12 @@ testios: echo "Cannot find a finalized iOS Python.framework. Have you run 'make install' to finalize the framework build?"; \ exit 1;\ fi - # Copy the testbed project into the build folder - cp -r $(srcdir)/iOS/testbed $(XCFOLDER) - # Copy the framework from the install location to the testbed project. - cp -r $(PYTHONFRAMEWORKPREFIX)/* $(XCFOLDER)/Python.xcframework/ios-arm64_x86_64-simulator - - # Run the test suite for the Xcode project, targeting the iOS simulator. - # If the suite fails, touch a file in the test folder as a marker - if ! xcodebuild test -project $(XCFOLDER)/iOSTestbed.xcodeproj -scheme "iOSTestbed" -destination "platform=iOS Simulator,name=iPhone SE (3rd Generation)" -resultBundlePath $(XCRESULT) -derivedDataPath $(XCFOLDER)/DerivedData ; then \ - touch $(XCFOLDER)/failed; \ - fi - # Regardless of success or failure, extract and print the test output - xcrun xcresulttool get --path $(XCRESULT) \ - --id $$( \ - xcrun xcresulttool get --path $(XCRESULT) --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['actions']['_values'][0]['actionResult']['logRef']['id']['_value'])" \ - ) \ - --format json | \ - $(PYTHON_FOR_BUILD) -c "import sys, json; result = json.load(sys.stdin); print(result['subsections']['_values'][1]['subsections']['_values'][0]['emittedOutput']['_value'])" + # Clone the testbed project into the XCFOLDER + $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" - @if test -e $(XCFOLDER)/failed ; then \ - exit 1; \ - fi + # Run the testbed project + $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run -- test -uall --single-process --rerun -W # Like test, but using --slow-ci which enables all test resources and use # longer timeout. Run an optional pybuildbot.identify script to include @@ -3053,8 +3049,7 @@ clean-retain-profile: pycremoval find build -name '*.py[co]' -exec rm -f {} ';' || true -rm -f pybuilddir.txt -rm -f _bootstrap_python - -rm -f python.html python*.js python.data python*.symbols python*.map - -rm -f $(WASM_STDLIB) + -rm -rf web_example python.mjs python.wasm python*.symbols python*.map -rm -f Programs/_testembed Programs/_freeze_module -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h diff --git a/Misc/ACKS b/Misc/ACKS index 08cd293eac3835..086930666822ad 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -24,6 +24,7 @@ Eitan Adler Anton Afanasyev Ali Afshar Nitika Agarwal +Maxim Ageev Anjani Agrawal Pablo S. Blum de Aguiar Jim Ahlstrom @@ -1154,6 +1155,7 @@ Mark Lutz Taras Lyapun Jim Lynch Mikael Lyngvig +Ilya Lyubavski Jeff MacDonald John Machin Andrew I MacIntyre @@ -1735,6 +1737,7 @@ Ng Pheng Siong Yann Sionneau George Sipe J. Sipprell +Giovanni Siragusa Ngalim Siregar Kragen Sitaker Kaartic Sivaraam @@ -2078,6 +2081,7 @@ Arnon Yaari Alakshendra Yadav Hirokazu Yamamoto Masayuki Yamamoto +Zhikang Yan Jingchen Ye Ka-Ping Yee Chi Hsuan Yen diff --git a/Misc/NEWS.d/3.14.0a2.rst b/Misc/NEWS.d/3.14.0a2.rst index 7384ce54cb8914..d82ec98b7a3c87 100644 --- a/Misc/NEWS.d/3.14.0a2.rst +++ b/Misc/NEWS.d/3.14.0a2.rst @@ -597,7 +597,7 @@ TypeError is now raised instead of ValueError for some logical errors. .. nonce: Jat5kq .. section: Library -Add :meth:`pathlib.Path.scandir` method to efficiently fetch directory +Add :meth:`!pathlib.Path.scandir` method to efficiently fetch directory children and their file attributes. This is a trivial wrapper of :func:`os.scandir`. diff --git a/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst new file mode 100644 index 00000000000000..d90067cd3bfaa3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-11-30-16-36-09.gh-issue-127111.QI9mMZ.rst @@ -0,0 +1,2 @@ +Updated the Emscripten web example to use ES6 modules and be built into a +distinct ``web_example`` subfolder. diff --git a/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst new file mode 100644 index 00000000000000..52ee84f5d6f6c3 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-06-12-47-52.gh-issue-127629.tD-ERQ.rst @@ -0,0 +1 @@ +Emscripten builds now include ctypes support. diff --git a/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst b/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst new file mode 100644 index 00000000000000..3fc1d8a1b51d30 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-12-12-17-21-45.gh-issue-127865.30GDzs.rst @@ -0,0 +1 @@ +Fix build failure on systems without thread-locals support. diff --git a/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst b/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst new file mode 100644 index 00000000000000..9d6b2e0c565623 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-12-10-15-19.gh-issue-109523.S2c3fi.rst @@ -0,0 +1 @@ +Reading text from a non-blocking stream with ``read`` may now raise a :exc:`BlockingIOError` if the operation cannot immediately return bytes. diff --git a/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst new file mode 100644 index 00000000000000..7254a04f61843d --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-08-27-09-07-56.gh-issue-123378.JJ6n_u.rst @@ -0,0 +1,6 @@ +Ensure that the value of :attr:`UnicodeEncodeError.start ` +retrieved by :c:func:`PyUnicodeEncodeError_GetStart` lies in +``[0, max(0, objlen - 1)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar +arguments apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` +and their corresponding C interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst b/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst new file mode 100644 index 00000000000000..8ea3c4ee2a2c53 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-11-26-22-06-10.gh-issue-127314.SsRrIu.rst @@ -0,0 +1,2 @@ +Improve error message when calling the C API without an active thread state +on the :term:`free-threaded ` build. diff --git a/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst b/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst new file mode 100644 index 00000000000000..107751579c4d91 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-02-16-10-36.gh-issue-123378.Q6YRwe.rst @@ -0,0 +1,6 @@ +Ensure that the value of :attr:`UnicodeEncodeError.end ` +retrieved by :c:func:`PyUnicodeEncodeError_GetEnd` lies in ``[min(1, objlen), +max(min(1, objlen), objlen)]`` where *objlen* is the length of +:attr:`UnicodeEncodeError.object `. Similar arguments +apply to :exc:`UnicodeDecodeError` and :exc:`UnicodeTranslateError` and their +corresponding C interface. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst b/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst new file mode 100644 index 00000000000000..70751f18f5cf17 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-12-10-14-25-22.gh-issue-127791.YRw4GU.rst @@ -0,0 +1,2 @@ +Fix loss of callbacks after more than one call to +:c:func:`PyUnstable_AtExit`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst new file mode 100644 index 00000000000000..7954c7f5927397 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst @@ -0,0 +1,2 @@ +Correct invalid corner cases which resulted in ``(nan+nanj)`` output in complex +multiplication, e.g., ``(1e300+1j)*(nan+infj)``. Patch by Sergey B Kirpichev. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst new file mode 100644 index 00000000000000..ef120802b5dc59 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-12-34-51.gh-issue-125420.jABXoZ.rst @@ -0,0 +1,2 @@ +Add :meth:`memoryview.count` to :class:`memoryview` objects. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst new file mode 100644 index 00000000000000..6ed823175f6754 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-14-13-28-16.gh-issue-125420.hNKixM.rst @@ -0,0 +1,2 @@ +Add :meth:`memoryview.index` to :class:`memoryview` objects. Patch by +Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst new file mode 100644 index 00000000000000..b41fff30433c34 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-10-27-04-47-28.gh-issue-126024.XCQSqT.rst @@ -0,0 +1,2 @@ +Optimize decoding of short UTF-8 sequences containing non-ASCII characters +by approximately 15%. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst new file mode 100644 index 00000000000000..13381c7630d7ce --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst @@ -0,0 +1 @@ +Fix crash in finalization of dtoa state. Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst new file mode 100644 index 00000000000000..8d7da0d4107021 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-17-21-35-55.gh-issue-126937.qluVM0.rst @@ -0,0 +1,3 @@ +Fix :exc:`TypeError` when a :class:`ctypes.Structure` has a field size +that doesn't fit into an unsigned 16-bit integer. +Instead, the maximum number of *bits* is :data:`sys.maxsize`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst new file mode 100644 index 00000000000000..9ef2b8dc33ed0f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-21-16-13-52.gh-issue-126491.0YvL94.rst @@ -0,0 +1,4 @@ +Add a marking phase to the GC. All objects that can be transitively reached +from builtin modules or the stacks are marked as reachable before cycle +detection. This reduces the amount of work done by the GC by approximately +half. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst new file mode 100644 index 00000000000000..6e2b87fe38146b --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-03-21-07-06.gh-issue-127536.3jMMrT.rst @@ -0,0 +1,2 @@ +Add missing locks around some list assignment operations in the free +threading build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst new file mode 100644 index 00000000000000..08b27a7890bb1c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-04-09-52-08.gh-issue-127434.RjkGT_.rst @@ -0,0 +1 @@ +The iOS compiler shims can now accept arguments with spaces. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst new file mode 100644 index 00000000000000..59491feeb9bcfa --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-05-19-25-00.gh-issue-127582.ogUY2a.rst @@ -0,0 +1,2 @@ +Fix non-thread-safe object resurrection when calling finalizers and watcher +callbacks in the free threading build. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst new file mode 100644 index 00000000000000..92b18b082eb30e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-06-01-09-40.gh-issue-127651.80cm6j.rst @@ -0,0 +1 @@ +When raising :exc:`ImportError` for missing symbols in ``from`` imports, use ``__file__`` in the error message if ``__spec__.origin`` is not a location diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst new file mode 100644 index 00000000000000..248e1b4855afb8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-09-11-29-10.gh-issue-127058.pqtBcZ.rst @@ -0,0 +1,3 @@ +``PySequence_Tuple`` now creates the resulting tuple atomically, preventing +partially created tuples being visible to the garbage collector or through +``gc.get_referrers()`` diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst new file mode 100644 index 00000000000000..f614dbb59bdc87 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-12-10-21-08-05.gh-issue-127740.0tWC9h.rst @@ -0,0 +1,3 @@ +Fix error message in :func:`bytes.fromhex` when given an odd number of +digits to properly indicate that an even number of hexadecimal digits is +required. diff --git a/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst b/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst new file mode 100644 index 00000000000000..79b3faa3d0d385 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2024-11-27-22-56-48.gh-issue-127347.xyddWS.rst @@ -0,0 +1 @@ +Publicly expose :func:`traceback.print_list` in :attr:`!traceback.__all__`. diff --git a/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst b/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst new file mode 100644 index 00000000000000..16ad75792aefa2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-07-30-11-37-40.gh-issue-122431.lAzVtu.rst @@ -0,0 +1 @@ +:func:`readline.append_history_file` now raises a :exc:`ValueError` when given a negative value. diff --git a/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst b/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst new file mode 100644 index 00000000000000..638f3f76239ca6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-08-27-18-58-01.gh-issue-123401.t4-FpI.rst @@ -0,0 +1,3 @@ +The :mod:`http.cookies` module now supports parsing obsolete :rfc:`850` +date formats, in accordance with :rfc:`9110` requirements. +Patch by Nano Zheng. diff --git a/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst b/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst new file mode 100644 index 00000000000000..ae1392e2caf387 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-04-22-02-30.gh-issue-85046.Y5d_ZN.rst @@ -0,0 +1 @@ +Add :data:`~errno.EHWPOISON` error code to :mod:`errno`. diff --git a/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst b/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst new file mode 100644 index 00000000000000..429fc2aa17bd42 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-13-10-44-25.gh-issue-126775.a3ubjh.rst @@ -0,0 +1 @@ +Make :func:`linecache.checkcache` thread safe and GC re-entrancy safe. diff --git a/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst b/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst new file mode 100644 index 00000000000000..c1a0ed6438135d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-16-10-52-48.gh-issue-126899.GFnfBt.rst @@ -0,0 +1,2 @@ +Make tkinter widget methods :meth:`!after` and :meth:`!after_idle` accept +arguments passed by keyword. diff --git a/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst b/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst new file mode 100644 index 00000000000000..b169c1508d0d30 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-18-23-18-27.gh-issue-112192.DRdRgP.rst @@ -0,0 +1 @@ +In the :mod:`trace` module, increase the coverage precision (``cov%``) to one decimal. diff --git a/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst b/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst new file mode 100644 index 00000000000000..7a0a7b7517b70d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-20-08-54-11.gh-issue-126618.ef_53g.rst @@ -0,0 +1,2 @@ +Fix the representation of :class:`itertools.count` objects when the count +value is :data:`sys.maxsize`. diff --git a/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst b/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst new file mode 100644 index 00000000000000..526785f68cc807 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-20-21-20-56.gh-issue-126992.RbU0FZ.rst @@ -0,0 +1 @@ +Fix LONG and INT opcodes to only use base 10 for string to integer conversion in :mod:`pickle`. diff --git a/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst b/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst new file mode 100644 index 00000000000000..8efe563f443774 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-21-06-03-46.gh-issue-127090.yUYwdh.rst @@ -0,0 +1,3 @@ +Fix value of :attr:`urllib.response.addinfourl.url` for ``file:`` URLs that +express relative paths and absolute Windows paths. The canonical URL generated +by :func:`urllib.request.pathname2url` is now used. diff --git a/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst b/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst new file mode 100644 index 00000000000000..0e4a03caf9f49d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-23-00-17-29.gh-issue-127221.OSXdFE.rst @@ -0,0 +1 @@ +Add colour to :mod:`unittest` output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst b/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst new file mode 100644 index 00000000000000..9fe7815e93cf4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-25-15-02-44.gh-issue-127255.UXeljc.rst @@ -0,0 +1,2 @@ +The :func:`~ctypes.CopyComPointer` function is now public. +Previously, this was private and only available in ``_ctypes``. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst new file mode 100644 index 00000000000000..58ebf5d0abe141 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst @@ -0,0 +1 @@ +Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`. diff --git a/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst b/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst new file mode 100644 index 00000000000000..a8c7b3d00755e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-27-17-04-38.gh-issue-59705.sAGyvs.rst @@ -0,0 +1,2 @@ +On Linux, :class:`threading.Thread` now sets the thread name to the +operating system. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst b/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst new file mode 100644 index 00000000000000..fb0380cba0b607 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-28-14-14-46.gh-issue-127257.n6-jU9.rst @@ -0,0 +1,2 @@ +In :mod:`ssl`, system call failures that OpenSSL reports using +``ERR_LIB_SYS`` are now raised as :exc:`OSError`. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst b/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst new file mode 100644 index 00000000000000..b56a77b4294ace --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-00-15-59.gh-issue-125413.WCN0vv.rst @@ -0,0 +1,3 @@ +Revert addition of :meth:`!pathlib.Path.scandir`. This method was added in +3.14.0a2. The optimizations remain for file system paths, but other +subclasses should only have to implement :meth:`pathlib.Path.iterdir`. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst b/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst new file mode 100644 index 00000000000000..2330fb66253265 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-14-45-26.gh-issue-127413.z11AUc.rst @@ -0,0 +1,2 @@ +Add the :option:`dis --specialized` command-line option to show specialized +bytecode. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst new file mode 100644 index 00000000000000..708c1a6437d812 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-29-23-02-43.gh-issue-127429.dQf2w4.rst @@ -0,0 +1,3 @@ +Fixed bug where, on cross-builds, the :mod:`sysconfig` POSIX data was being +generated with the host Python's ``Makefile``. The data is now generated from +current build's ``Makefile``. diff --git a/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst b/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst new file mode 100644 index 00000000000000..69b6ce68a47509 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-30-21-46-15.gh-issue-127321.M78fBv.rst @@ -0,0 +1 @@ +:func:`pdb.set_trace` will not stop at an opcode that does not have an associated line number anymore. diff --git a/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst new file mode 100644 index 00000000000000..03d6953b9ddfa1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-01-22-28-41.gh-issue-127065.tFpRer.rst @@ -0,0 +1 @@ +Make :func:`operator.methodcaller` thread-safe and re-entrant safe. diff --git a/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst b/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst new file mode 100644 index 00000000000000..8ada0b57ddc257 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-01-23-18-43.gh-issue-127481.K36AoP.rst @@ -0,0 +1 @@ +Add the ``EPOLLWAKEUP`` constant to the :mod:`select` module. diff --git a/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst b/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst new file mode 100644 index 00000000000000..e245fa2bdd00b4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-11-01-16.gh-issue-93312.9sB-Qw.rst @@ -0,0 +1,2 @@ +Include ```` to get ``os.PIDFD_NONBLOCK`` constant. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst new file mode 100644 index 00000000000000..677acf5baab3fa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-04-15-04-12.gh-issue-126821.lKCLVV.rst @@ -0,0 +1,2 @@ +macOS and iOS apps can now choose to redirect stdout and stderr to the +system log during interpreter configuration. diff --git a/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst b/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst new file mode 100644 index 00000000000000..48a6c7d30b4a26 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-05-10-14-52.gh-issue-127627.fgCHOZ.rst @@ -0,0 +1,2 @@ +Added ``posix._emscripten_debugger()`` to help with debugging the test suite on +the Emscripten target. diff --git a/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst b/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst new file mode 100644 index 00000000000000..58769029d79977 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-06-17-28-55.gh-issue-127610.ctv_NP.rst @@ -0,0 +1,3 @@ +Added validation for more than one var-positional or +var-keyword parameters in :class:`inspect.Signature`. +Patch by Maxim Ageev. diff --git a/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst b/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst new file mode 100644 index 00000000000000..6c1b7bef610e8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-07-15-28-31.gh-issue-127718.9dpLfi.rst @@ -0,0 +1 @@ +Add colour to :mod:`test.regrtest` output. Patch by Hugo van Kemenade. diff --git a/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst b/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst new file mode 100644 index 00000000000000..417e9ac986f27a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-07-23-06-44.gh-issue-126789.4dqfV1.rst @@ -0,0 +1,5 @@ +Fixed :func:`sysconfig.get_config_vars`, :func:`sysconfig.get_paths`, and +siblings, returning outdated cached data if the value of :data:`sys.prefix` +or :data:`sys.exec_prefix` changes. Overwriting :data:`sys.prefix` or +:data:`sys.exec_prefix` still is discouraged, as that might break other +parts of the code. diff --git a/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst b/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst new file mode 100644 index 00000000000000..44821300f6e4e6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-12-08-08-36-18.gh-issue-127732.UEKxoa.rst @@ -0,0 +1 @@ +The :mod:`platform` module now correctly detects Windows Server 2025. diff --git a/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst b/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst new file mode 100644 index 00000000000000..76cfc58121d3bd --- /dev/null +++ b/Misc/NEWS.d/next/Security/2024-12-05-21-35-19.gh-issue-127655.xpPoOf.rst @@ -0,0 +1 @@ +Fixed the :class:`!asyncio.selector_events._SelectorSocketTransport` transport not pausing writes for the protocol when the buffer reaches the high water mark when using :meth:`asyncio.WriteTransport.writelines`. diff --git a/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst new file mode 100644 index 00000000000000..fb307c7cb9bf1d --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-04-15-03-24.gh-issue-126925.uxAMK-.rst @@ -0,0 +1,2 @@ +iOS test results are now streamed during test execution, and the deprecated +xcresulttool is no longer used. diff --git a/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst b/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst new file mode 100644 index 00000000000000..ac5d9827b07199 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2024-12-09-12-35-44.gh-issue-127637.KLx-9I.rst @@ -0,0 +1 @@ +Add tests for the :mod:`dis` command-line interface. Patch by Bénédikt Tran. diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 3220852c8398e0..2b9e8a1a10d6f5 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -110,10 +110,16 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, goto error; } - Py_ssize_t bit_size = NUM_BITS(size); - if (bit_size) { + if (bit_size_obj != Py_None) { +#ifdef Py_DEBUG + Py_ssize_t bit_size = NUM_BITS(size); assert(bit_size > 0); assert(bit_size <= info->size * 8); + // Currently, the bit size is specified redundantly + // in NUM_BITS(size) and bit_size_obj. + // Verify that they match. + assert(PyLong_AsSsize_t(bit_size_obj) == bit_size); +#endif switch(info->ffi_type_pointer.type) { case FFI_TYPE_UINT8: case FFI_TYPE_UINT16: diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index 5dbbe0b3285d58..5ca5b62427600d 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -292,7 +292,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!tmp) { goto error; } - Py_ssize_t total_align = PyLong_AsInt(tmp); + Py_ssize_t total_align = PyLong_AsSsize_t(tmp); Py_DECREF(tmp); if (total_align < 0) { if (!PyErr_Occurred()) { @@ -306,7 +306,7 @@ PyCStructUnionType_update_stginfo(PyObject *type, PyObject *fields, int isStruct if (!tmp) { goto error; } - Py_ssize_t total_size = PyLong_AsInt(tmp); + Py_ssize_t total_size = PyLong_AsSsize_t(tmp); Py_DECREF(tmp); if (total_size < 0) { if (!PyErr_Occurred()) { diff --git a/Modules/_io/clinic/bufferedio.c.h b/Modules/_io/clinic/bufferedio.c.h index b703d7e6855398..e035bd99baca5f 100644 --- a/Modules/_io/clinic/bufferedio.c.h +++ b/Modules/_io/clinic/bufferedio.c.h @@ -330,9 +330,7 @@ _io__Buffered_simple_flush(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io__Buffered_closed_HAS_DOCSTR) -# define _io__Buffered_closed_DOCSTR _io__Buffered_closed__doc__ -#else +#if !defined(_io__Buffered_closed_DOCSTR) # define _io__Buffered_closed_DOCSTR NULL #endif #if defined(_IO__BUFFERED_CLOSED_GETSETDEF) @@ -472,9 +470,7 @@ _io__Buffered_writable(buffered *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io__Buffered_name_HAS_DOCSTR) -# define _io__Buffered_name_DOCSTR _io__Buffered_name__doc__ -#else +#if !defined(_io__Buffered_name_DOCSTR) # define _io__Buffered_name_DOCSTR NULL #endif #if defined(_IO__BUFFERED_NAME_GETSETDEF) @@ -499,9 +495,7 @@ _io__Buffered_name_get(buffered *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io__Buffered_mode_HAS_DOCSTR) -# define _io__Buffered_mode_DOCSTR _io__Buffered_mode__doc__ -#else +#if !defined(_io__Buffered_mode_DOCSTR) # define _io__Buffered_mode_DOCSTR NULL #endif #if defined(_IO__BUFFERED_MODE_GETSETDEF) @@ -1252,4 +1246,4 @@ _io_BufferedRandom___init__(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=36abca5bd2f63ea1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8f28a97987a9fbe1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index ac8b4b7b85b76d..6f9205af32f010 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -476,9 +476,7 @@ _io_StringIO___setstate__(stringio *self, PyObject *state) return return_value; } -#if defined(_io_StringIO_closed_HAS_DOCSTR) -# define _io_StringIO_closed_DOCSTR _io_StringIO_closed__doc__ -#else +#if !defined(_io_StringIO_closed_DOCSTR) # define _io_StringIO_closed_DOCSTR NULL #endif #if defined(_IO_STRINGIO_CLOSED_GETSETDEF) @@ -503,9 +501,7 @@ _io_StringIO_closed_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_StringIO_line_buffering_HAS_DOCSTR) -# define _io_StringIO_line_buffering_DOCSTR _io_StringIO_line_buffering__doc__ -#else +#if !defined(_io_StringIO_line_buffering_DOCSTR) # define _io_StringIO_line_buffering_DOCSTR NULL #endif #if defined(_IO_STRINGIO_LINE_BUFFERING_GETSETDEF) @@ -530,9 +526,7 @@ _io_StringIO_line_buffering_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_StringIO_newlines_HAS_DOCSTR) -# define _io_StringIO_newlines_DOCSTR _io_StringIO_newlines__doc__ -#else +#if !defined(_io_StringIO_newlines_DOCSTR) # define _io_StringIO_newlines_DOCSTR NULL #endif #if defined(_IO_STRINGIO_NEWLINES_GETSETDEF) @@ -556,4 +550,4 @@ _io_StringIO_newlines_get(stringio *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=8c8d4f8fa32986bb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d2b092274469d42 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index c9301c5a23fa86..160f80ada43660 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -208,11 +208,9 @@ PyDoc_STRVAR(_io__TextIOBase_encoding__doc__, "Encoding of the text stream.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_encoding_HAS_DOCSTR +#define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#if defined(_io__TextIOBase_encoding_HAS_DOCSTR) -# define _io__TextIOBase_encoding_DOCSTR _io__TextIOBase_encoding__doc__ -#else +#if !defined(_io__TextIOBase_encoding_DOCSTR) # define _io__TextIOBase_encoding_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ENCODING_GETSETDEF) @@ -237,11 +235,9 @@ PyDoc_STRVAR(_io__TextIOBase_newlines__doc__, "Only line endings translated during reading are considered.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_newlines_HAS_DOCSTR +#define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#if defined(_io__TextIOBase_newlines_HAS_DOCSTR) -# define _io__TextIOBase_newlines_DOCSTR _io__TextIOBase_newlines__doc__ -#else +#if !defined(_io__TextIOBase_newlines_DOCSTR) # define _io__TextIOBase_newlines_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_NEWLINES_GETSETDEF) @@ -264,11 +260,9 @@ PyDoc_STRVAR(_io__TextIOBase_errors__doc__, "The error setting of the decoder or encoder.\n" "\n" "Subclasses should override."); -#define _io__TextIOBase_errors_HAS_DOCSTR +#define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#if defined(_io__TextIOBase_errors_HAS_DOCSTR) -# define _io__TextIOBase_errors_DOCSTR _io__TextIOBase_errors__doc__ -#else +#if !defined(_io__TextIOBase_errors_DOCSTR) # define _io__TextIOBase_errors_DOCSTR NULL #endif #if defined(_IO__TEXTIOBASE_ERRORS_GETSETDEF) @@ -1138,9 +1132,7 @@ _io_TextIOWrapper_close(textio *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_io_TextIOWrapper_name_HAS_DOCSTR) -# define _io_TextIOWrapper_name_DOCSTR _io_TextIOWrapper_name__doc__ -#else +#if !defined(_io_TextIOWrapper_name_DOCSTR) # define _io_TextIOWrapper_name_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NAME_GETSETDEF) @@ -1165,9 +1157,7 @@ _io_TextIOWrapper_name_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_closed_HAS_DOCSTR) -# define _io_TextIOWrapper_closed_DOCSTR _io_TextIOWrapper_closed__doc__ -#else +#if !defined(_io_TextIOWrapper_closed_DOCSTR) # define _io_TextIOWrapper_closed_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_CLOSED_GETSETDEF) @@ -1192,9 +1182,7 @@ _io_TextIOWrapper_closed_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_newlines_HAS_DOCSTR) -# define _io_TextIOWrapper_newlines_DOCSTR _io_TextIOWrapper_newlines__doc__ -#else +#if !defined(_io_TextIOWrapper_newlines_DOCSTR) # define _io_TextIOWrapper_newlines_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_NEWLINES_GETSETDEF) @@ -1219,9 +1207,7 @@ _io_TextIOWrapper_newlines_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper_errors_HAS_DOCSTR) -# define _io_TextIOWrapper_errors_DOCSTR _io_TextIOWrapper_errors__doc__ -#else +#if !defined(_io_TextIOWrapper_errors_DOCSTR) # define _io_TextIOWrapper_errors_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER_ERRORS_GETSETDEF) @@ -1246,9 +1232,7 @@ _io_TextIOWrapper_errors_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_io_TextIOWrapper__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1273,9 +1257,7 @@ _io_TextIOWrapper__CHUNK_SIZE_get(textio *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_HAS_DOCSTR) -# define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR _io_TextIOWrapper__CHUNK_SIZE__doc__ -#else +#if !defined(_io_TextIOWrapper__CHUNK_SIZE_DOCSTR) # define _io_TextIOWrapper__CHUNK_SIZE_DOCSTR NULL #endif #if defined(_IO_TEXTIOWRAPPER__CHUNK_SIZE_GETSETDEF) @@ -1299,4 +1281,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(textio *self, PyObject *value, void *Py_UNUSED return return_value; } -/*[clinic end generated code: output=459c0e50acd772b1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1172c500a022c65d input=a9049054013a1b77]*/ diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 0d851ee211511c..791ee070401fe5 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -1992,6 +1992,12 @@ _io_TextIOWrapper_read_impl(textio *self, Py_ssize_t n) if (bytes == NULL) goto fail; + if (bytes == Py_None){ + Py_DECREF(bytes); + PyErr_SetString(PyExc_BlockingIOError, "Read returned None."); + return NULL; + } + _PyIO_State *state = self->state; if (Py_IS_TYPE(self->decoder, state->PyIncrementalNewlineDecoder_Type)) decoded = _PyIncrementalNewlineDecoder_decode(self->decoder, diff --git a/Modules/_operator.c b/Modules/_operator.c index 6c1945174ab7cd..ce3ef015710223 100644 --- a/Modules/_operator.c +++ b/Modules/_operator.c @@ -1595,78 +1595,75 @@ static PyType_Spec attrgetter_type_spec = { typedef struct { PyObject_HEAD PyObject *name; - PyObject *xargs; // reference to arguments passed in constructor + PyObject *args; PyObject *kwds; - PyObject **vectorcall_args; /* Borrowed references */ + PyObject *vectorcall_args; PyObject *vectorcall_kwnames; vectorcallfunc vectorcall; } methodcallerobject; -#ifndef Py_GIL_DISABLED -static int _methodcaller_initialize_vectorcall(methodcallerobject* mc) -{ - PyObject* args = mc->xargs; - PyObject* kwds = mc->kwds; - - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - assert(nargs > 0); - mc->vectorcall_args = PyMem_Calloc( - nargs + (kwds ? PyDict_Size(kwds) : 0), - sizeof(PyObject*)); - if (!mc->vectorcall_args) { - PyErr_NoMemory(); - return -1; - } - /* The first item of vectorcall_args will be filled with obj later */ - if (nargs > 1) { - memcpy(mc->vectorcall_args, PySequence_Fast_ITEMS(args), - nargs * sizeof(PyObject*)); - } - if (kwds) { - const Py_ssize_t nkwds = PyDict_Size(kwds); - - mc->vectorcall_kwnames = PyTuple_New(nkwds); - if (!mc->vectorcall_kwnames) { - return -1; - } - Py_ssize_t i = 0, ppos = 0; - PyObject* key, * value; - while (PyDict_Next(kwds, &ppos, &key, &value)) { - PyTuple_SET_ITEM(mc->vectorcall_kwnames, i, Py_NewRef(key)); - mc->vectorcall_args[nargs + i] = value; // borrowed reference - ++i; - } - } - else { - mc->vectorcall_kwnames = NULL; - } - return 1; -} +#define _METHODCALLER_MAX_ARGS 8 static PyObject * -methodcaller_vectorcall( - methodcallerobject *mc, PyObject *const *args, size_t nargsf, PyObject* kwnames) +methodcaller_vectorcall(methodcallerobject *mc, PyObject *const *args, + size_t nargsf, PyObject* kwnames) { if (!_PyArg_CheckPositional("methodcaller", PyVectorcall_NARGS(nargsf), 1, 1) || !_PyArg_NoKwnames("methodcaller", kwnames)) { return NULL; } - if (mc->vectorcall_args == NULL) { - if (_methodcaller_initialize_vectorcall(mc) < 0) { - return NULL; - } - } + assert(mc->vectorcall_args != NULL); + + PyObject *tmp_args[_METHODCALLER_MAX_ARGS]; + tmp_args[0] = args[0]; + assert(1 + PyTuple_GET_SIZE(mc->vectorcall_args) <= _METHODCALLER_MAX_ARGS); + memcpy(tmp_args + 1, _PyTuple_ITEMS(mc->vectorcall_args), sizeof(PyObject *) * PyTuple_GET_SIZE(mc->vectorcall_args)); - assert(mc->vectorcall_args != 0); - mc->vectorcall_args[0] = args[0]; - return PyObject_VectorcallMethod( - mc->name, mc->vectorcall_args, - (PyTuple_GET_SIZE(mc->xargs)) | PY_VECTORCALL_ARGUMENTS_OFFSET, + return PyObject_VectorcallMethod(mc->name, tmp_args, + (1 + PyTuple_GET_SIZE(mc->args)) | PY_VECTORCALL_ARGUMENTS_OFFSET, mc->vectorcall_kwnames); } -#endif +static int +_methodcaller_initialize_vectorcall(methodcallerobject* mc) +{ + PyObject* args = mc->args; + PyObject* kwds = mc->kwds; + + if (kwds && PyDict_Size(kwds)) { + PyObject *values = PyDict_Values(kwds); + if (!values) { + return -1; + } + PyObject *values_tuple = PySequence_Tuple(values); + Py_DECREF(values); + if (!values_tuple) { + return -1; + } + if (PyTuple_GET_SIZE(args)) { + mc->vectorcall_args = PySequence_Concat(args, values_tuple); + Py_DECREF(values_tuple); + if (mc->vectorcall_args == NULL) { + return -1; + } + } + else { + mc->vectorcall_args = values_tuple; + } + mc->vectorcall_kwnames = PySequence_Tuple(kwds); + if (!mc->vectorcall_kwnames) { + return -1; + } + } + else { + mc->vectorcall_args = Py_NewRef(args); + mc->vectorcall_kwnames = NULL; + } + + mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; + return 0; +} /* AC 3.5: variable number of arguments, not currently support by AC */ static PyObject * @@ -1694,25 +1691,30 @@ methodcaller_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (mc == NULL) { return NULL; } + mc->vectorcall = NULL; + mc->vectorcall_args = NULL; + mc->vectorcall_kwnames = NULL; + mc->kwds = Py_XNewRef(kwds); Py_INCREF(name); PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternMortal(interp, &name); mc->name = name; - mc->xargs = Py_XNewRef(args); // allows us to use borrowed references - mc->kwds = Py_XNewRef(kwds); - mc->vectorcall_args = 0; - + mc->args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + if (mc->args == NULL) { + Py_DECREF(mc); + return NULL; + } -#ifdef Py_GIL_DISABLED - // gh-127065: The current implementation of methodcaller_vectorcall - // is not thread-safe because it modifies the `vectorcall_args` array, - // which is shared across calls. - mc->vectorcall = NULL; -#else - mc->vectorcall = (vectorcallfunc)methodcaller_vectorcall; -#endif + Py_ssize_t vectorcall_size = PyTuple_GET_SIZE(args) + + (kwds ? PyDict_Size(kwds) : 0); + if (vectorcall_size < (_METHODCALLER_MAX_ARGS)) { + if (_methodcaller_initialize_vectorcall(mc) < 0) { + Py_DECREF(mc); + return NULL; + } + } PyObject_GC_Track(mc); return (PyObject *)mc; @@ -1722,13 +1724,10 @@ static void methodcaller_clear(methodcallerobject *mc) { Py_CLEAR(mc->name); - Py_CLEAR(mc->xargs); + Py_CLEAR(mc->args); Py_CLEAR(mc->kwds); - if (mc->vectorcall_args != NULL) { - PyMem_Free(mc->vectorcall_args); - mc->vectorcall_args = 0; - Py_CLEAR(mc->vectorcall_kwnames); - } + Py_CLEAR(mc->vectorcall_args); + Py_CLEAR(mc->vectorcall_kwnames); } static void @@ -1745,8 +1744,10 @@ static int methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) { Py_VISIT(mc->name); - Py_VISIT(mc->xargs); + Py_VISIT(mc->args); Py_VISIT(mc->kwds); + Py_VISIT(mc->vectorcall_args); + Py_VISIT(mc->vectorcall_kwnames); Py_VISIT(Py_TYPE(mc)); return 0; } @@ -1765,15 +1766,7 @@ methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) if (method == NULL) return NULL; - - PyObject *cargs = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (cargs == NULL) { - Py_DECREF(method); - return NULL; - } - - result = PyObject_Call(method, cargs, mc->kwds); - Py_DECREF(cargs); + result = PyObject_Call(method, mc->args, mc->kwds); Py_DECREF(method); return result; } @@ -1791,7 +1784,7 @@ methodcaller_repr(methodcallerobject *mc) } numkwdargs = mc->kwds != NULL ? PyDict_GET_SIZE(mc->kwds) : 0; - numposargs = PyTuple_GET_SIZE(mc->xargs) - 1; + numposargs = PyTuple_GET_SIZE(mc->args); numtotalargs = numposargs + numkwdargs; if (numtotalargs == 0) { @@ -1807,7 +1800,7 @@ methodcaller_repr(methodcallerobject *mc) } for (i = 0; i < numposargs; ++i) { - PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->xargs, i+1)); + PyObject *onerepr = PyObject_Repr(PyTuple_GET_ITEM(mc->args, i)); if (onerepr == NULL) goto done; PyTuple_SET_ITEM(argreprs, i, onerepr); @@ -1859,14 +1852,14 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) { if (!mc->kwds || PyDict_GET_SIZE(mc->kwds) == 0) { Py_ssize_t i; - Py_ssize_t newarg_size = PyTuple_GET_SIZE(mc->xargs); - PyObject *newargs = PyTuple_New(newarg_size); + Py_ssize_t callargcount = PyTuple_GET_SIZE(mc->args); + PyObject *newargs = PyTuple_New(1 + callargcount); if (newargs == NULL) return NULL; PyTuple_SET_ITEM(newargs, 0, Py_NewRef(mc->name)); - for (i = 1; i < newarg_size; ++i) { - PyObject *arg = PyTuple_GET_ITEM(mc->xargs, i); - PyTuple_SET_ITEM(newargs, i, Py_NewRef(arg)); + for (i = 0; i < callargcount; ++i) { + PyObject *arg = PyTuple_GET_ITEM(mc->args, i); + PyTuple_SET_ITEM(newargs, i + 1, Py_NewRef(arg)); } return Py_BuildValue("ON", Py_TYPE(mc), newargs); } @@ -1884,12 +1877,7 @@ methodcaller_reduce(methodcallerobject *mc, PyObject *Py_UNUSED(ignored)) constructor = PyObject_VectorcallDict(partial, newargs, 2, mc->kwds); Py_DECREF(partial); - PyObject *args = PyTuple_GetSlice(mc->xargs, 1, PyTuple_GET_SIZE(mc->xargs)); - if (!args) { - Py_DECREF(constructor); - return NULL; - } - return Py_BuildValue("NO", constructor, args); + return Py_BuildValue("NO", constructor, mc->args); } } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index 2696f38046121f..599b5f92c2a1f7 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -5211,16 +5211,14 @@ load_int(PickleState *state, UnpicklerObject *self) return bad_readline(state); errno = 0; - /* XXX: Should the base argument of strtol() be explicitly set to 10? - XXX(avassalotti): Should this uses PyOS_strtol()? */ - x = strtol(s, &endptr, 0); + /* XXX(avassalotti): Should this uses PyOS_strtol()? */ + x = strtol(s, &endptr, 10); if (errno || (*endptr != '\n' && *endptr != '\0')) { /* Hm, maybe we've got something long. Let's try reading * it as a Python int object. */ errno = 0; - /* XXX: Same thing about the base here. */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) { PyErr_SetString(PyExc_ValueError, "could not convert string to int"); @@ -5370,8 +5368,7 @@ load_long(PickleState *state, UnpicklerObject *self) the 'L' to be present. */ if (s[len-2] == 'L') s[len-2] = '\0'; - /* XXX: Should the base argument explicitly set to 10? */ - value = PyLong_FromString(s, NULL, 0); + value = PyLong_FromString(s, NULL, 10); if (value == NULL) return -1; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index e5b8bf21002ea5..e7df132869fee6 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -667,6 +667,11 @@ PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) ERR_GET_REASON(e) == SSL_R_CERTIFICATE_VERIFY_FAILED) { type = state->PySSLCertVerificationErrorObject; } + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } p = PY_SSL_ERROR_SYSCALL; } break; @@ -692,6 +697,11 @@ PySSL_SetError(PySSLSocket *sslsock, const char *filename, int lineno) errstr = "EOF occurred in violation of protocol"; } #endif + if (ERR_GET_LIB(e) == ERR_LIB_SYS) { + // A system error is being reported; reason is set to errno + errno = ERR_GET_REASON(e); + return PyErr_SetFromErrno(PyExc_OSError); + } break; } default: @@ -2162,11 +2172,17 @@ _ssl__SSLSocket_compression_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.context + +This changes the context associated with the SSLSocket. + +This is typically used from within a callback function set by the sni_callback +on the SSLContext to change the certificate information associated with the +SSLSocket before the cryptographic exchange handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=25aa82e4d9fa344a]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ { return Py_NewRef(self->ctx); } @@ -2233,11 +2249,15 @@ _ssl__SSLSocket_server_hostname_get_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.owner + +The Python-level owner of this object. + +Passed as "self" in servername callback. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_owner_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=1f278cb930382927 input=bc2861ff3cf1402e]*/ +/*[clinic end generated code: output=1f278cb930382927 input=978a8382d9c25c92]*/ { if (self->owner == NULL) { Py_RETURN_NONE; @@ -2851,11 +2871,13 @@ _ssl__SSLSocket_verify_client_post_handshake_impl(PySSLSocket *self) @critical_section @getter _ssl._SSLSocket.session + +The underlying SSLSession object. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_session_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=a5cd5755b35da670 input=b9792df9255a9f63]*/ +/*[clinic end generated code: output=a5cd5755b35da670 input=d427318604244bf8]*/ { /* get_session can return sessions from a server-side connection, * it does not check for handshake done or client socket. */ @@ -3657,11 +3679,13 @@ _ssl__SSLContext_maximum_version_set_impl(PySSLContext *self, @critical_section @getter _ssl._SSLContext.num_tickets + +Control the number of TLSv1.3 session tickets. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_num_tickets_get_impl(PySSLContext *self) -/*[clinic end generated code: output=3d06d016318846c9 input=1dee26d75163c073]*/ +/*[clinic end generated code: output=3d06d016318846c9 input=1e2599a2e22564ff]*/ { // Clinic seems to be misbehaving when the comment is wrapped with in directive #if defined(TLS1_3_VERSION) && !defined(OPENSSL_NO_TLS1_3) @@ -3705,11 +3729,13 @@ _ssl__SSLContext_num_tickets_set_impl(PySSLContext *self, PyObject *value) @critical_section @getter _ssl._SSLContext.security_level + +The current security level. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_security_level_get_impl(PySSLContext *self) -/*[clinic end generated code: output=56ece09e6a9572d0 input=a0416598e07c3183]*/ +/*[clinic end generated code: output=56ece09e6a9572d0 input=2bdeecb57bb86e3f]*/ { PyObject *res = PyLong_FromLong(SSL_CTX_get_security_level(self->ctx)); return res; @@ -4666,11 +4692,18 @@ _servername_callback(SSL *s, int *al, void *args) @critical_section @getter _ssl._SSLContext.sni_callback + +Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. + +If the argument is None then the callback is disabled. The method is called +with the SSLSocket, the server name as a string, and the SSLContext object. + +See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=22dd28c31fdc4318]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -5243,11 +5276,13 @@ memory_bio_dealloc(PySSLMemoryBIO *self) @critical_section @getter _ssl.MemoryBIO.pending + +The number of bytes pending in the memory BIO. [clinic start generated code]*/ static PyObject * _ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=19236a32a51ac8ff input=c0b6d14eba107f6a]*/ +/*[clinic end generated code: output=19236a32a51ac8ff input=02d9063d8ac31732]*/ { size_t res = BIO_ctrl_pending(self->bio); return PyLong_FromSize_t(res); @@ -5257,11 +5292,13 @@ _ssl_MemoryBIO_pending_get_impl(PySSLMemoryBIO *self) @critical_section @getter _ssl.MemoryBIO.eof + +Whether the memory BIO is at EOF. [clinic start generated code]*/ static PyObject * _ssl_MemoryBIO_eof_get_impl(PySSLMemoryBIO *self) -/*[clinic end generated code: output=c255a9ea16e31b92 input=0f5c6be69752e04c]*/ +/*[clinic end generated code: output=c255a9ea16e31b92 input=c6ecc12c4509de1f]*/ { size_t pending = BIO_ctrl_pending(self->bio); return PyBool_FromLong((pending == 0) && self->eof_written); @@ -5502,11 +5539,13 @@ PySSLSession_clear(PySSLSession *self) @critical_section @getter _ssl.SSLSession.time + +Session creation time (seconds since epoch). [clinic start generated code]*/ static PyObject * _ssl_SSLSession_time_get_impl(PySSLSession *self) -/*[clinic end generated code: output=4b887b9299de9be4 input=8d1e4afd09103279]*/ +/*[clinic end generated code: output=4b887b9299de9be4 input=67f2325284450ae2]*/ { #if OPENSSL_VERSION_NUMBER >= 0x30300000L return _PyLong_FromTime_t(SSL_SESSION_get_time_ex(self->session)); @@ -5519,11 +5558,13 @@ _ssl_SSLSession_time_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.timeout + +Session timeout (delta in seconds). [clinic start generated code]*/ static PyObject * _ssl_SSLSession_timeout_get_impl(PySSLSession *self) -/*[clinic end generated code: output=82339c148ab2f7d1 input=ae5e84a9d85df60d]*/ +/*[clinic end generated code: output=82339c148ab2f7d1 input=cd17c2b087c442f2]*/ { long timeout = SSL_SESSION_get_timeout(self->session); PyObject *res = PyLong_FromLong(timeout); @@ -5534,11 +5575,13 @@ _ssl_SSLSession_timeout_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.ticket_lifetime_hint + +Ticket life time hint. [clinic start generated code]*/ static PyObject * _ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) -/*[clinic end generated code: output=c8b6db498136c275 input=d0e06942ddd8d07f]*/ +/*[clinic end generated code: output=c8b6db498136c275 input=f0e2df50961a7806]*/ { unsigned long hint = SSL_SESSION_get_ticket_lifetime_hint(self->session); return PyLong_FromUnsignedLong(hint); @@ -5548,11 +5591,13 @@ _ssl_SSLSession_ticket_lifetime_hint_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.id + +Session ID. [clinic start generated code]*/ static PyObject * _ssl_SSLSession_id_get_impl(PySSLSession *self) -/*[clinic end generated code: output=c532fb96b10c5adf input=e7322372cf6325dd]*/ +/*[clinic end generated code: output=c532fb96b10c5adf input=0a379e64312b776d]*/ { const unsigned char *id; @@ -5565,11 +5610,13 @@ _ssl_SSLSession_id_get_impl(PySSLSession *self) @critical_section @getter _ssl.SSLSession.has_ticket + +Does the session contain a ticket? [clinic start generated code]*/ static PyObject * _ssl_SSLSession_has_ticket_get_impl(PySSLSession *self) -/*[clinic end generated code: output=aa3ccfc40b10b96d input=1a48ae8955fa9601]*/ +/*[clinic end generated code: output=aa3ccfc40b10b96d input=fa475555f53a5086]*/ { int res = SSL_SESSION_has_ticket(self->session); return res ? Py_True : Py_False; diff --git a/Modules/_testcapi/exceptions.c b/Modules/_testcapi/exceptions.c index 316ef0e7ad7e55..e92d9670e7c792 100644 --- a/Modules/_testcapi/exceptions.c +++ b/Modules/_testcapi/exceptions.c @@ -359,6 +359,161 @@ _testcapi_unstable_exc_prep_reraise_star_impl(PyObject *module, return PyUnstable_Exc_PrepReraiseStar(orig, excs); } +/* Test PyUnicodeEncodeError_GetStart */ +static PyObject * +unicode_encode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeEncodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeDecodeError_GetStart */ +static PyObject * +unicode_decode_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeDecodeError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeTranslateError_GetStart */ +static PyObject * +unicode_translate_get_start(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t start; + if (PyUnicodeTranslateError_GetStart(arg, &start) < 0) { + return NULL; + } + RETURN_SIZE(start); +} + +/* Test PyUnicodeEncodeError_SetStart */ +static PyObject * +unicode_encode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetStart */ +static PyObject * +unicode_decode_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetStart */ +static PyObject * +unicode_translate_set_start(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t start; + if (PyArg_ParseTuple(args, "On", &exc, &start) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetStart(exc, start) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeEncodeError_GetEnd */ +static PyObject * +unicode_encode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeEncodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeDecodeError_GetEnd */ +static PyObject * +unicode_decode_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeDecodeError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeTranslateError_GetEnd */ +static PyObject * +unicode_translate_get_end(PyObject *Py_UNUSED(module), PyObject *arg) +{ + Py_ssize_t end; + if (PyUnicodeTranslateError_GetEnd(arg, &end) < 0) { + return NULL; + } + RETURN_SIZE(end); +} + +/* Test PyUnicodeEncodeError_SetEnd */ +static PyObject * +unicode_encode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeEncodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeDecodeError_SetEnd */ +static PyObject * +unicode_decode_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeDecodeError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +/* Test PyUnicodeTranslateError_SetEnd */ +static PyObject * +unicode_translate_set_end(PyObject *Py_UNUSED(module), PyObject *args) +{ + PyObject *exc; + Py_ssize_t end; + if (PyArg_ParseTuple(args, "On", &exc, &end) < 0) { + return NULL; + } + if (PyUnicodeTranslateError_SetEnd(exc, end) < 0) { + return NULL; + } + Py_RETURN_NONE; +} /* * Define the PyRecurdingInfinitelyError_Type @@ -403,6 +558,18 @@ static PyMethodDef test_methods[] = { _TESTCAPI_SET_EXCEPTION_METHODDEF _TESTCAPI_TRACEBACK_PRINT_METHODDEF _TESTCAPI_UNSTABLE_EXC_PREP_RERAISE_STAR_METHODDEF + {"unicode_encode_get_start", unicode_encode_get_start, METH_O}, + {"unicode_decode_get_start", unicode_decode_get_start, METH_O}, + {"unicode_translate_get_start", unicode_translate_get_start, METH_O}, + {"unicode_encode_set_start", unicode_encode_set_start, METH_VARARGS}, + {"unicode_decode_set_start", unicode_decode_set_start, METH_VARARGS}, + {"unicode_translate_set_start", unicode_translate_set_start, METH_VARARGS}, + {"unicode_encode_get_end", unicode_encode_get_end, METH_O}, + {"unicode_decode_get_end", unicode_decode_get_end, METH_O}, + {"unicode_translate_get_end", unicode_translate_get_end, METH_O}, + {"unicode_encode_set_end", unicode_encode_set_end, METH_VARARGS}, + {"unicode_decode_set_end", unicode_decode_set_end, METH_VARARGS}, + {"unicode_translate_set_end", unicode_translate_set_end, METH_VARARGS}, {NULL}, }; diff --git a/Modules/_testcapi/structmember.c b/Modules/_testcapi/structmember.c index 096eaecd40855f..ef30a5a9944e3c 100644 --- a/Modules/_testcapi/structmember.c +++ b/Modules/_testcapi/structmember.c @@ -22,6 +22,7 @@ typedef struct { char inplace_member[6]; long long longlong_member; unsigned long long ulonglong_member; + char char_member; } all_structmembers; typedef struct { @@ -46,6 +47,7 @@ static struct PyMemberDef test_members_newapi[] = { {"T_STRING_INPLACE", Py_T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", Py_T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", Py_T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", Py_T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; @@ -56,9 +58,9 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "T_BOOL", "T_BYTE", "T_UBYTE", "T_SHORT", "T_USHORT", "T_INT", "T_UINT", "T_LONG", "T_ULONG", "T_PYSSIZET", "T_FLOAT", "T_DOUBLE", "T_STRING_INPLACE", - "T_LONGLONG", "T_ULONGLONG", + "T_LONGLONG", "T_ULONGLONG", "T_CHAR", NULL}; - static const char fmt[] = "|bbBhHiIlknfds#LK"; + static const char fmt[] = "|bbBhHiIlknfds#LKc"; test_structmembers *ob; const char *s = NULL; Py_ssize_t string_len = 0; @@ -82,7 +84,8 @@ test_structmembers_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) &ob->structmembers.double_member, &s, &string_len, &ob->structmembers.longlong_member, - &ob->structmembers.ulonglong_member)) + &ob->structmembers.ulonglong_member, + &ob->structmembers.char_member)) { Py_DECREF(ob); return NULL; @@ -132,6 +135,7 @@ static struct PyMemberDef test_members[] = { {"T_STRING_INPLACE", T_STRING_INPLACE, offsetof(test_structmembers, structmembers.inplace_member), 0, NULL}, {"T_LONGLONG", T_LONGLONG, offsetof(test_structmembers, structmembers.longlong_member), 0, NULL}, {"T_ULONGLONG", T_ULONGLONG, offsetof(test_structmembers, structmembers.ulonglong_member), 0, NULL}, + {"T_CHAR", T_CHAR, offsetof(test_structmembers, structmembers.char_member), 0, NULL}, {NULL} }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 26f68691e44f83..8d86b535effb9a 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -3353,6 +3353,53 @@ type_freeze(PyObject *module, PyObject *args) Py_RETURN_NONE; } +struct atexit_data { + int called; + PyThreadState *tstate; + PyInterpreterState *interp; +}; + +static void +atexit_callback(void *data) +{ + struct atexit_data *at_data = (struct atexit_data *)data; + // Ensure that the callback is from the same interpreter + assert(PyThreadState_Get() == at_data->tstate); + assert(PyInterpreterState_Get() == at_data->interp); + ++at_data->called; +} + +static PyObject * +test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *oldts = PyThreadState_Swap(NULL); + PyThreadState *tstate = Py_NewInterpreter(); + + struct atexit_data data = {0}; + data.tstate = PyThreadState_Get(); + data.interp = PyInterpreterState_Get(); + + int amount = 10; + for (int i = 0; i < amount; ++i) + { + int res = PyUnstable_AtExit(tstate->interp, atexit_callback, (void *)&data); + if (res < 0) { + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + PyErr_SetString(PyExc_RuntimeError, "atexit callback failed"); + return NULL; + } + } + + Py_EndInterpreter(tstate); + PyThreadState_Swap(oldts); + + if (data.called != amount) { + PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); + return NULL; + } + Py_RETURN_NONE; +} static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, @@ -3495,6 +3542,7 @@ static PyMethodDef TestMethods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"finalize_thread_hang", finalize_thread_hang, METH_O, NULL}, {"type_freeze", type_freeze, METH_VARARGS}, + {"test_atexit", test_atexit, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index a925191d479bd6..014f89997f7f60 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1236,39 +1236,6 @@ unicode_transformdecimalandspacetoascii(PyObject *self, PyObject *arg) return _PyUnicode_TransformDecimalAndSpaceToASCII(arg); } - -struct atexit_data { - int called; -}; - -static void -callback(void *data) -{ - ((struct atexit_data *)data)->called += 1; -} - -static PyObject * -test_atexit(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyThreadState *oldts = PyThreadState_Swap(NULL); - PyThreadState *tstate = Py_NewInterpreter(); - - struct atexit_data data = {0}; - int res = PyUnstable_AtExit(tstate->interp, callback, (void *)&data); - Py_EndInterpreter(tstate); - PyThreadState_Swap(oldts); - if (res < 0) { - return NULL; - } - - if (data.called == 0) { - PyErr_SetString(PyExc_RuntimeError, "atexit callback not called"); - return NULL; - } - Py_RETURN_NONE; -} - - static PyObject * test_pyobject_is_freed(const char *test_name, PyObject *op) { @@ -2076,6 +2043,20 @@ has_deferred_refcount(PyObject *self, PyObject *op) return PyBool_FromLong(_PyObject_HasDeferredRefcount(op)); } +static PyObject * +get_tracked_heap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromInt64(PyInterpreterState_Get()->gc.heap_size); +} + +static PyObject * +is_static_immortal(PyObject *self, PyObject *op) +{ + if (_Py_IsStaticImmortal(op)) { + Py_RETURN_TRUE; + } + Py_RETURN_FALSE; +} static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, @@ -2123,7 +2104,6 @@ static PyMethodDef module_functions[] = { {"_PyTraceMalloc_GetTraceback", tracemalloc_get_traceback, METH_VARARGS}, {"test_tstate_capi", test_tstate_capi, METH_NOARGS, NULL}, {"_PyUnicode_TransformDecimalAndSpaceToASCII", unicode_transformdecimalandspacetoascii, METH_O}, - {"test_atexit", test_atexit, METH_NOARGS}, {"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS}, {"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS}, @@ -2174,6 +2154,8 @@ static PyMethodDef module_functions[] = { {"get_static_builtin_types", get_static_builtin_types, METH_NOARGS}, {"identify_type_slot_wrappers", identify_type_slot_wrappers, METH_NOARGS}, {"has_deferred_refcount", has_deferred_refcount, METH_O}, + {"get_tracked_heap_size", get_tracked_heap_size, METH_NOARGS}, + {"is_static_immortal", is_static_immortal, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 4a45445e2f62db..75b34a8df7622c 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -17,6 +17,8 @@ # include // SIGINT #endif +#include "clinic/_threadmodule.c.h" + // ThreadError is just an alias to PyExc_RuntimeError #define ThreadError PyExc_RuntimeError @@ -44,6 +46,13 @@ get_thread_state(PyObject *module) return (thread_module_state *)state; } + +/*[clinic input] +module _thread +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=be8dbe5cc4b16df7]*/ + + // _ThreadHandle type // Handles state transitions according to the following diagram: @@ -2354,6 +2363,95 @@ PyDoc_STRVAR(thread__get_main_thread_ident_doc, Internal only. Return a non-zero integer that uniquely identifies the main thread\n\ of the main interpreter."); + +#ifdef HAVE_PTHREAD_GETNAME_NP +/*[clinic input] +_thread._get_name + +Get the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread__get_name_impl(PyObject *module) +/*[clinic end generated code: output=20026e7ee3da3dd7 input=35cec676833d04c8]*/ +{ + // Linux and macOS are limited to respectively 16 and 64 bytes + char name[100]; + pthread_t thread = pthread_self(); + int rc = pthread_getname_np(thread, name, Py_ARRAY_LENGTH(name)); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + +#ifdef __sun + return PyUnicode_DecodeUTF8(name, strlen(name), "surrogateescape"); +#else + return PyUnicode_DecodeFSDefault(name); +#endif +} +#endif // HAVE_PTHREAD_GETNAME_NP + + +#ifdef HAVE_PTHREAD_SETNAME_NP +/*[clinic input] +_thread.set_name + + name as name_obj: unicode + +Set the name of the current thread. +[clinic start generated code]*/ + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj) +/*[clinic end generated code: output=402b0c68e0c0daed input=7e7acd98261be82f]*/ +{ +#ifdef __sun + // Solaris always uses UTF-8 + const char *encoding = "utf-8"; +#else + // Encode the thread name to the filesystem encoding using the "replace" + // error handler + PyInterpreterState *interp = _PyInterpreterState_GET(); + const char *encoding = interp->unicode.fs_codec.encoding; +#endif + PyObject *name_encoded; + name_encoded = PyUnicode_AsEncodedString(name_obj, encoding, "replace"); + if (name_encoded == NULL) { + return NULL; + } + +#ifdef PYTHREAD_NAME_MAXLEN + // Truncate to PYTHREAD_NAME_MAXLEN bytes + the NUL byte if needed + if (PyBytes_GET_SIZE(name_encoded) > PYTHREAD_NAME_MAXLEN) { + PyObject *truncated; + truncated = PyBytes_FromStringAndSize(PyBytes_AS_STRING(name_encoded), + PYTHREAD_NAME_MAXLEN); + if (truncated == NULL) { + Py_DECREF(name_encoded); + return NULL; + } + Py_SETREF(name_encoded, truncated); + } +#endif + + const char *name = PyBytes_AS_STRING(name_encoded); +#ifdef __APPLE__ + int rc = pthread_setname_np(name); +#else + pthread_t thread = pthread_self(); + int rc = pthread_setname_np(thread, name); +#endif + Py_DECREF(name_encoded); + if (rc) { + errno = rc; + return PyErr_SetFromErrno(PyExc_OSError); + } + Py_RETURN_NONE; +} +#endif // HAVE_PTHREAD_SETNAME_NP + + static PyMethodDef thread_methods[] = { {"start_new_thread", (PyCFunction)thread_PyThread_start_new_thread, METH_VARARGS, start_new_thread_doc}, @@ -2393,6 +2491,8 @@ static PyMethodDef thread_methods[] = { METH_O, thread__make_thread_handle_doc}, {"_get_main_thread_ident", thread__get_main_thread_ident, METH_NOARGS, thread__get_main_thread_ident_doc}, + _THREAD_SET_NAME_METHODDEF + _THREAD__GET_NAME_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -2484,6 +2584,13 @@ thread_module_exec(PyObject *module) llist_init(&state->shutdown_handles); +#ifdef PYTHREAD_NAME_MAXLEN + if (PyModule_AddIntConstant(module, "_NAME_MAXLEN", + PYTHREAD_NAME_MAXLEN) < 0) { + return -1; + } +#endif + return 0; } diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 297a8d74ba3bf4..c009235b7a36c2 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -27,7 +27,10 @@ int PyUnstable_AtExit(PyInterpreterState *interp, atexit_datacallbackfunc func, void *data) { - assert(interp == _PyInterpreterState_GET()); + PyThreadState *tstate = _PyThreadState_GET(); + _Py_EnsureTstateNotNULL(tstate); + assert(tstate->interp == interp); + atexit_callback *callback = PyMem_Malloc(sizeof(atexit_callback)); if (callback == NULL) { PyErr_NoMemory(); @@ -38,12 +41,13 @@ PyUnstable_AtExit(PyInterpreterState *interp, callback->next = NULL; struct atexit_state *state = &interp->atexit; - if (state->ll_callbacks == NULL) { + atexit_callback *top = state->ll_callbacks; + if (top == NULL) { state->ll_callbacks = callback; - state->last_ll_callback = callback; } else { - state->last_ll_callback->next = callback; + callback->next = top; + state->ll_callbacks = callback; } return 0; } diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index 582eef16c13244..1ff85e32ffe5a0 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -258,9 +258,15 @@ _ssl__SSLSocket_compression(PySSLSocket *self, PyObject *Py_UNUSED(ignored)) return _ssl__SSLSocket_compression_impl(self); } -#if defined(_ssl__SSLSocket_context_HAS_DOCSTR) -# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ -#else +PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, +"This changes the context associated with the SSLSocket.\n" +"\n" +"This is typically used from within a callback function set by the sni_callback\n" +"on the SSLContext to change the certificate information associated with the\n" +"SSLSocket before the cryptographic exchange handshake messages."); +#define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ + +#if !defined(_ssl__SSLSocket_context_DOCSTR) # define _ssl__SSLSocket_context_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) @@ -285,9 +291,7 @@ _ssl__SSLSocket_context_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_CONTEXT_HAS_DOCSTR) -# define _ssl__SSLSocket_context_DOCSTR _ssl__SSLSocket_context__doc__ -#else +#if !defined(_ssl__SSLSocket_context_DOCSTR) # define _ssl__SSLSocket_context_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_CONTEXT_GETSETDEF) @@ -314,11 +318,9 @@ _ssl__SSLSocket_context_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_server_side__doc__, "Whether this is a server-side socket."); -#define _ssl__SSLSocket_server_side_HAS_DOCSTR +#define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ -#if defined(_ssl__SSLSocket_server_side_HAS_DOCSTR) -# define _ssl__SSLSocket_server_side_DOCSTR _ssl__SSLSocket_server_side__doc__ -#else +#if !defined(_ssl__SSLSocket_server_side_DOCSTR) # define _ssl__SSLSocket_server_side_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SERVER_SIDE_GETSETDEF) @@ -345,11 +347,9 @@ _ssl__SSLSocket_server_side_get(PySSLSocket *self, void *Py_UNUSED(context)) PyDoc_STRVAR(_ssl__SSLSocket_server_hostname__doc__, "The currently set server hostname (for SNI)."); -#define _ssl__SSLSocket_server_hostname_HAS_DOCSTR +#define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ -#if defined(_ssl__SSLSocket_server_hostname_HAS_DOCSTR) -# define _ssl__SSLSocket_server_hostname_DOCSTR _ssl__SSLSocket_server_hostname__doc__ -#else +#if !defined(_ssl__SSLSocket_server_hostname_DOCSTR) # define _ssl__SSLSocket_server_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SERVER_HOSTNAME_GETSETDEF) @@ -374,9 +374,13 @@ _ssl__SSLSocket_server_hostname_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl__SSLSocket_owner_HAS_DOCSTR) -# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ -#else +PyDoc_STRVAR(_ssl__SSLSocket_owner__doc__, +"The Python-level owner of this object.\n" +"\n" +"Passed as \"self\" in servername callback."); +#define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ + +#if !defined(_ssl__SSLSocket_owner_DOCSTR) # define _ssl__SSLSocket_owner_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) @@ -401,9 +405,7 @@ _ssl__SSLSocket_owner_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_OWNER_HAS_DOCSTR) -# define _ssl__SSLSocket_owner_DOCSTR _ssl__SSLSocket_owner__doc__ -#else +#if !defined(_ssl__SSLSocket_owner_DOCSTR) # define _ssl__SSLSocket_owner_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_OWNER_GETSETDEF) @@ -664,9 +666,11 @@ _ssl__SSLSocket_verify_client_post_handshake(PySSLSocket *self, PyObject *Py_UNU return return_value; } -#if defined(_ssl__SSLSocket_session_HAS_DOCSTR) -# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ -#else +PyDoc_STRVAR(_ssl__SSLSocket_session__doc__, +"The underlying SSLSession object."); +#define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ + +#if !defined(_ssl__SSLSocket_session_DOCSTR) # define _ssl__SSLSocket_session_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) @@ -691,9 +695,7 @@ _ssl__SSLSocket_session_get(PySSLSocket *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLSOCKET_SESSION_HAS_DOCSTR) -# define _ssl__SSLSocket_session_DOCSTR _ssl__SSLSocket_session__doc__ -#else +#if !defined(_ssl__SSLSocket_session_DOCSTR) # define _ssl__SSLSocket_session_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_GETSETDEF) @@ -720,11 +722,9 @@ _ssl__SSLSocket_session_set(PySSLSocket *self, PyObject *value, void *Py_UNUSED( PyDoc_STRVAR(_ssl__SSLSocket_session_reused__doc__, "Was the client session reused during handshake?"); -#define _ssl__SSLSocket_session_reused_HAS_DOCSTR +#define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ -#if defined(_ssl__SSLSocket_session_reused_HAS_DOCSTR) -# define _ssl__SSLSocket_session_reused_DOCSTR _ssl__SSLSocket_session_reused__doc__ -#else +#if !defined(_ssl__SSLSocket_session_reused_DOCSTR) # define _ssl__SSLSocket_session_reused_DOCSTR NULL #endif #if defined(_SSL__SSLSOCKET_SESSION_REUSED_GETSETDEF) @@ -873,9 +873,7 @@ _ssl__SSLContext__set_alpn_protocols(PySSLContext *self, PyObject *arg) return return_value; } -#if defined(_ssl__SSLContext_verify_mode_HAS_DOCSTR) -# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ -#else +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) # define _ssl__SSLContext_verify_mode_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) @@ -900,9 +898,7 @@ _ssl__SSLContext_verify_mode_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_VERIFY_MODE_HAS_DOCSTR) -# define _ssl__SSLContext_verify_mode_DOCSTR _ssl__SSLContext_verify_mode__doc__ -#else +#if !defined(_ssl__SSLContext_verify_mode_DOCSTR) # define _ssl__SSLContext_verify_mode_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_MODE_GETSETDEF) @@ -927,9 +923,7 @@ _ssl__SSLContext_verify_mode_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_verify_flags_HAS_DOCSTR) -# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ -#else +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) # define _ssl__SSLContext_verify_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) @@ -954,9 +948,7 @@ _ssl__SSLContext_verify_flags_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_HAS_DOCSTR) -# define _ssl__SSLContext_verify_flags_DOCSTR _ssl__SSLContext_verify_flags__doc__ -#else +#if !defined(_ssl__SSLContext_verify_flags_DOCSTR) # define _ssl__SSLContext_verify_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_VERIFY_FLAGS_GETSETDEF) @@ -981,9 +973,7 @@ _ssl__SSLContext_verify_flags_set(PySSLContext *self, PyObject *value, void *Py_ return return_value; } -#if defined(_ssl__SSLContext_minimum_version_HAS_DOCSTR) -# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ -#else +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) # define _ssl__SSLContext_minimum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) @@ -1008,9 +998,7 @@ _ssl__SSLContext_minimum_version_get(PySSLContext *self, void *Py_UNUSED(context return return_value; } -#if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_HAS_DOCSTR) -# define _ssl__SSLContext_minimum_version_DOCSTR _ssl__SSLContext_minimum_version__doc__ -#else +#if !defined(_ssl__SSLContext_minimum_version_DOCSTR) # define _ssl__SSLContext_minimum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MINIMUM_VERSION_GETSETDEF) @@ -1036,9 +1024,7 @@ _ssl__SSLContext_minimum_version_set(PySSLContext *self, PyObject *value, void * return return_value; } -#if defined(_ssl__SSLContext_maximum_version_HAS_DOCSTR) -# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ -#else +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) # define _ssl__SSLContext_maximum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) @@ -1063,9 +1049,7 @@ _ssl__SSLContext_maximum_version_get(PySSLContext *self, void *Py_UNUSED(context return return_value; } -#if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_HAS_DOCSTR) -# define _ssl__SSLContext_maximum_version_DOCSTR _ssl__SSLContext_maximum_version__doc__ -#else +#if !defined(_ssl__SSLContext_maximum_version_DOCSTR) # define _ssl__SSLContext_maximum_version_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_MAXIMUM_VERSION_GETSETDEF) @@ -1091,9 +1075,11 @@ _ssl__SSLContext_maximum_version_set(PySSLContext *self, PyObject *value, void * return return_value; } -#if defined(_ssl__SSLContext_num_tickets_HAS_DOCSTR) -# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ -#else +PyDoc_STRVAR(_ssl__SSLContext_num_tickets__doc__, +"Control the number of TLSv1.3 session tickets."); +#define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ + +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) # define _ssl__SSLContext_num_tickets_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) @@ -1118,9 +1104,7 @@ _ssl__SSLContext_num_tickets_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_NUM_TICKETS_HAS_DOCSTR) -# define _ssl__SSLContext_num_tickets_DOCSTR _ssl__SSLContext_num_tickets__doc__ -#else +#if !defined(_ssl__SSLContext_num_tickets_DOCSTR) # define _ssl__SSLContext_num_tickets_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_NUM_TICKETS_GETSETDEF) @@ -1145,9 +1129,11 @@ _ssl__SSLContext_num_tickets_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_security_level_HAS_DOCSTR) -# define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ -#else +PyDoc_STRVAR(_ssl__SSLContext_security_level__doc__, +"The current security level."); +#define _ssl__SSLContext_security_level_DOCSTR _ssl__SSLContext_security_level__doc__ + +#if !defined(_ssl__SSLContext_security_level_DOCSTR) # define _ssl__SSLContext_security_level_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SECURITY_LEVEL_GETSETDEF) @@ -1172,9 +1158,7 @@ _ssl__SSLContext_security_level_get(PySSLContext *self, void *Py_UNUSED(context) return return_value; } -#if defined(_ssl__SSLContext_options_HAS_DOCSTR) -# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ -#else +#if !defined(_ssl__SSLContext_options_DOCSTR) # define _ssl__SSLContext_options_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) @@ -1199,9 +1183,7 @@ _ssl__SSLContext_options_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_OPTIONS_HAS_DOCSTR) -# define _ssl__SSLContext_options_DOCSTR _ssl__SSLContext_options__doc__ -#else +#if !defined(_ssl__SSLContext_options_DOCSTR) # define _ssl__SSLContext_options_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_OPTIONS_GETSETDEF) @@ -1226,9 +1208,7 @@ _ssl__SSLContext_options_set(PySSLContext *self, PyObject *value, void *Py_UNUSE return return_value; } -#if defined(_ssl__SSLContext__host_flags_HAS_DOCSTR) -# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ -#else +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) # define _ssl__SSLContext__host_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) @@ -1253,9 +1233,7 @@ _ssl__SSLContext__host_flags_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT__HOST_FLAGS_HAS_DOCSTR) -# define _ssl__SSLContext__host_flags_DOCSTR _ssl__SSLContext__host_flags__doc__ -#else +#if !defined(_ssl__SSLContext__host_flags_DOCSTR) # define _ssl__SSLContext__host_flags_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT__HOST_FLAGS_GETSETDEF) @@ -1280,9 +1258,7 @@ _ssl__SSLContext__host_flags_set(PySSLContext *self, PyObject *value, void *Py_U return return_value; } -#if defined(_ssl__SSLContext_check_hostname_HAS_DOCSTR) -# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ -#else +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) # define _ssl__SSLContext_check_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) @@ -1307,9 +1283,7 @@ _ssl__SSLContext_check_hostname_get(PySSLContext *self, void *Py_UNUSED(context) return return_value; } -#if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_HAS_DOCSTR) -# define _ssl__SSLContext_check_hostname_DOCSTR _ssl__SSLContext_check_hostname__doc__ -#else +#if !defined(_ssl__SSLContext_check_hostname_DOCSTR) # define _ssl__SSLContext_check_hostname_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_CHECK_HOSTNAME_GETSETDEF) @@ -1334,9 +1308,7 @@ _ssl__SSLContext_check_hostname_set(PySSLContext *self, PyObject *value, void *P return return_value; } -#if defined(_ssl__SSLContext_protocol_HAS_DOCSTR) -# define _ssl__SSLContext_protocol_DOCSTR _ssl__SSLContext_protocol__doc__ -#else +#if !defined(_ssl__SSLContext_protocol_DOCSTR) # define _ssl__SSLContext_protocol_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_PROTOCOL_GETSETDEF) @@ -1799,9 +1771,16 @@ _ssl__SSLContext_set_ecdh_curve(PySSLContext *self, PyObject *name) return return_value; } -#if defined(_ssl__SSLContext_sni_callback_HAS_DOCSTR) -# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ -#else +PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, +"Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" +"\n" +"If the argument is None then the callback is disabled. The method is called\n" +"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"\n" +"See RFC 6066 for details of the SNI extension."); +#define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ + +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) # define _ssl__SSLContext_sni_callback_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) @@ -1826,9 +1805,7 @@ _ssl__SSLContext_sni_callback_get(PySSLContext *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_HAS_DOCSTR) -# define _ssl__SSLContext_sni_callback_DOCSTR _ssl__SSLContext_sni_callback__doc__ -#else +#if !defined(_ssl__SSLContext_sni_callback_DOCSTR) # define _ssl__SSLContext_sni_callback_DOCSTR NULL #endif #if defined(_SSL__SSLCONTEXT_SNI_CALLBACK_GETSETDEF) @@ -2121,9 +2098,11 @@ _ssl_MemoryBIO(PyTypeObject *type, PyObject *args, PyObject *kwargs) return return_value; } -#if defined(_ssl_MemoryBIO_pending_HAS_DOCSTR) -# define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ -#else +PyDoc_STRVAR(_ssl_MemoryBIO_pending__doc__, +"The number of bytes pending in the memory BIO."); +#define _ssl_MemoryBIO_pending_DOCSTR _ssl_MemoryBIO_pending__doc__ + +#if !defined(_ssl_MemoryBIO_pending_DOCSTR) # define _ssl_MemoryBIO_pending_DOCSTR NULL #endif #if defined(_SSL_MEMORYBIO_PENDING_GETSETDEF) @@ -2148,9 +2127,11 @@ _ssl_MemoryBIO_pending_get(PySSLMemoryBIO *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_MemoryBIO_eof_HAS_DOCSTR) -# define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ -#else +PyDoc_STRVAR(_ssl_MemoryBIO_eof__doc__, +"Whether the memory BIO is at EOF."); +#define _ssl_MemoryBIO_eof_DOCSTR _ssl_MemoryBIO_eof__doc__ + +#if !defined(_ssl_MemoryBIO_eof_DOCSTR) # define _ssl_MemoryBIO_eof_DOCSTR NULL #endif #if defined(_SSL_MEMORYBIO_EOF_GETSETDEF) @@ -2279,9 +2260,11 @@ _ssl_MemoryBIO_write_eof(PySSLMemoryBIO *self, PyObject *Py_UNUSED(ignored)) return return_value; } -#if defined(_ssl_SSLSession_time_HAS_DOCSTR) -# define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ -#else +PyDoc_STRVAR(_ssl_SSLSession_time__doc__, +"Session creation time (seconds since epoch)."); +#define _ssl_SSLSession_time_DOCSTR _ssl_SSLSession_time__doc__ + +#if !defined(_ssl_SSLSession_time_DOCSTR) # define _ssl_SSLSession_time_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TIME_GETSETDEF) @@ -2306,9 +2289,11 @@ _ssl_SSLSession_time_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_timeout_HAS_DOCSTR) -# define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ -#else +PyDoc_STRVAR(_ssl_SSLSession_timeout__doc__, +"Session timeout (delta in seconds)."); +#define _ssl_SSLSession_timeout_DOCSTR _ssl_SSLSession_timeout__doc__ + +#if !defined(_ssl_SSLSession_timeout_DOCSTR) # define _ssl_SSLSession_timeout_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TIMEOUT_GETSETDEF) @@ -2333,9 +2318,11 @@ _ssl_SSLSession_timeout_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_ticket_lifetime_hint_HAS_DOCSTR) -# define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ -#else +PyDoc_STRVAR(_ssl_SSLSession_ticket_lifetime_hint__doc__, +"Ticket life time hint."); +#define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR _ssl_SSLSession_ticket_lifetime_hint__doc__ + +#if !defined(_ssl_SSLSession_ticket_lifetime_hint_DOCSTR) # define _ssl_SSLSession_ticket_lifetime_hint_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_TICKET_LIFETIME_HINT_GETSETDEF) @@ -2360,9 +2347,11 @@ _ssl_SSLSession_ticket_lifetime_hint_get(PySSLSession *self, void *Py_UNUSED(con return return_value; } -#if defined(_ssl_SSLSession_id_HAS_DOCSTR) -# define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ -#else +PyDoc_STRVAR(_ssl_SSLSession_id__doc__, +"Session ID."); +#define _ssl_SSLSession_id_DOCSTR _ssl_SSLSession_id__doc__ + +#if !defined(_ssl_SSLSession_id_DOCSTR) # define _ssl_SSLSession_id_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_ID_GETSETDEF) @@ -2387,9 +2376,11 @@ _ssl_SSLSession_id_get(PySSLSession *self, void *Py_UNUSED(context)) return return_value; } -#if defined(_ssl_SSLSession_has_ticket_HAS_DOCSTR) -# define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ -#else +PyDoc_STRVAR(_ssl_SSLSession_has_ticket__doc__, +"Does the session contain a ticket?"); +#define _ssl_SSLSession_has_ticket_DOCSTR _ssl_SSLSession_has_ticket__doc__ + +#if !defined(_ssl_SSLSession_has_ticket_DOCSTR) # define _ssl_SSLSession_has_ticket_DOCSTR NULL #endif #if defined(_SSL_SSLSESSION_HAS_TICKET_GETSETDEF) @@ -2839,4 +2830,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=4c2af0c8fab7ec4e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=654d6d7af659f6cd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_threadmodule.c.h b/Modules/clinic/_threadmodule.c.h new file mode 100644 index 00000000000000..8f0507d40285b3 --- /dev/null +++ b/Modules/clinic/_threadmodule.c.h @@ -0,0 +1,104 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +#if defined(HAVE_PTHREAD_GETNAME_NP) + +PyDoc_STRVAR(_thread__get_name__doc__, +"_get_name($module, /)\n" +"--\n" +"\n" +"Get the name of the current thread."); + +#define _THREAD__GET_NAME_METHODDEF \ + {"_get_name", (PyCFunction)_thread__get_name, METH_NOARGS, _thread__get_name__doc__}, + +static PyObject * +_thread__get_name_impl(PyObject *module); + +static PyObject * +_thread__get_name(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _thread__get_name_impl(module); +} + +#endif /* defined(HAVE_PTHREAD_GETNAME_NP) */ + +#if defined(HAVE_PTHREAD_SETNAME_NP) + +PyDoc_STRVAR(_thread_set_name__doc__, +"set_name($module, /, name)\n" +"--\n" +"\n" +"Set the name of the current thread."); + +#define _THREAD_SET_NAME_METHODDEF \ + {"set_name", _PyCFunction_CAST(_thread_set_name), METH_FASTCALL|METH_KEYWORDS, _thread_set_name__doc__}, + +static PyObject * +_thread_set_name_impl(PyObject *module, PyObject *name_obj); + +static PyObject * +_thread_set_name(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(name), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"name", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "set_name", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + PyObject *name_obj; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("set_name", "argument 'name'", "str", args[0]); + goto exit; + } + name_obj = args[0]; + return_value = _thread_set_name_impl(module, name_obj); + +exit: + return return_value; +} + +#endif /* defined(HAVE_PTHREAD_SETNAME_NP) */ + +#ifndef _THREAD__GET_NAME_METHODDEF + #define _THREAD__GET_NAME_METHODDEF +#endif /* !defined(_THREAD__GET_NAME_METHODDEF) */ + +#ifndef _THREAD_SET_NAME_METHODDEF + #define _THREAD_SET_NAME_METHODDEF +#endif /* !defined(_THREAD_SET_NAME_METHODDEF) */ +/*[clinic end generated code: output=b5cb85aaccc45bf6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index cd0c4faeac83d1..554299b8598299 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -12447,6 +12447,28 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) return os__create_environ_impl(module); } +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_debugger__doc__, +"_emscripten_debugger($module, /)\n" +"--\n" +"\n" +"Create a breakpoint for the JavaScript debugger. Emscripten only."); + +#define OS__EMSCRIPTEN_DEBUGGER_METHODDEF \ + {"_emscripten_debugger", (PyCFunction)os__emscripten_debugger, METH_NOARGS, os__emscripten_debugger__doc__}, + +static PyObject * +os__emscripten_debugger_impl(PyObject *module); + +static PyObject * +os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return os__emscripten_debugger_impl(module); +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13114,4 +13136,8 @@ os__create_environ(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=7ee14f5e880092f5 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF + #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ +/*[clinic end generated code: output=9c2ca1dbf986c62c input=a9049054013a1b77]*/ diff --git a/Modules/errnomodule.c b/Modules/errnomodule.c index 3f96f2f846d612..9557d68e759497 100644 --- a/Modules/errnomodule.c +++ b/Modules/errnomodule.c @@ -845,6 +845,9 @@ errno_exec(PyObject *module) #ifdef ENOKEY add_errcode("ENOKEY", ENOKEY, "Required key not available"); #endif +#ifdef EHWPOISON + add_errcode("EHWPOISON", EHWPOISON, "Memory page has hardware error"); +#endif #ifdef EKEYEXPIRED add_errcode("EKEYEXPIRED", EKEYEXPIRED, "Key has expired"); #endif diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 78fbdcdf77a923..3f736f0cf19968 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -3235,7 +3235,7 @@ typedef struct { fast_mode: when cnt an integer < PY_SSIZE_T_MAX and no step is specified. - assert(cnt != PY_SSIZE_T_MAX && long_cnt == NULL && long_step==PyLong(1)); + assert(long_cnt == NULL && long_step==PyLong(1)); Advances with: cnt += 1 When count hits PY_SSIZE_T_MAX, switch to slow_mode. @@ -3291,9 +3291,6 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, PyErr_Clear(); fast_mode = 0; } - else if (cnt == PY_SSIZE_T_MAX) { - fast_mode = 0; - } } } else { cnt = 0; @@ -3325,7 +3322,7 @@ itertools_count_impl(PyTypeObject *type, PyObject *long_cnt, else cnt = PY_SSIZE_T_MAX; - assert((cnt != PY_SSIZE_T_MAX && long_cnt == NULL && fast_mode) || + assert((long_cnt == NULL && fast_mode) || (cnt == PY_SSIZE_T_MAX && long_cnt != NULL && !fast_mode)); assert(!fast_mode || (PyLong_Check(long_step) && PyLong_AS_LONG(long_step) == 1)); @@ -3418,7 +3415,7 @@ count_next(countobject *lz) static PyObject * count_repr(countobject *lz) { - if (lz->cnt != PY_SSIZE_T_MAX) + if (lz->long_cnt == NULL) return PyUnicode_FromFormat("%s(%zd)", _PyType_Name(Py_TYPE(lz)), lz->cnt); diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6eb7054b566e3f..2045c6065b8e7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -73,6 +73,9 @@ #ifdef HAVE_SYS_TIME_H # include // futimes() #endif +#ifdef HAVE_SYS_PIDFD_H +# include // PIDFD_NONBLOCK +#endif // SGI apparently needs this forward declaration @@ -81,6 +84,9 @@ extern char * _getpty(int *, int, mode_t, int); #endif +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" // emscripten_debugger() +#endif /* * A number of APIs are available on macOS from a certain macOS version. @@ -16842,8 +16848,24 @@ os__create_environ_impl(PyObject *module) } -static PyMethodDef posix_methods[] = { +#ifdef __EMSCRIPTEN__ +/*[clinic input] +os._emscripten_debugger +Create a breakpoint for the JavaScript debugger. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_debugger_impl(PyObject *module) +/*[clinic end generated code: output=ad47dc3bf0661343 input=d814b1877fb6083a]*/ +{ + emscripten_debugger(); + Py_RETURN_NONE; +} +#endif /* __EMSCRIPTEN__ */ + + +static PyMethodDef posix_methods[] = { OS_STAT_METHODDEF OS_ACCESS_METHODDEF OS_TTYNAME_METHODDEF @@ -17057,6 +17079,7 @@ static PyMethodDef posix_methods[] = { OS__INPUTHOOK_METHODDEF OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF + OS__EMSCRIPTEN_DEBUGGER_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/readline.c b/Modules/readline.c index 35655c70a4618f..7d1f703f7dbdde 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -351,6 +351,12 @@ readline_append_history_file_impl(PyObject *module, int nelements, PyObject *filename_obj) /*[clinic end generated code: output=5df06fc9da56e4e4 input=784b774db3a4b7c5]*/ { + if (nelements < 0) + { + PyErr_SetString(PyExc_ValueError, "nelements must be positive"); + return NULL; + } + PyObject *filename_bytes; const char *filename; int err; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index 6ced71cca73178..e14e114a6dafd0 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2715,6 +2715,10 @@ _select_exec(PyObject *m) #ifdef EPOLLMSG ADD_INT(EPOLLMSG); #endif +#ifdef EPOLLWAKEUP + /* Kernel 3.5+ */ + ADD_INT(EPOLLWAKEUP); +#endif #ifdef EPOLL_CLOEXEC ADD_INT(EPOLL_CLOEXEC); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index ca839dc55e0519..b13e6a9de10114 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -71,13 +71,13 @@ typedef struct { static SHA3object * newSHA3object(PyTypeObject *type) { - SHA3object *newobj; - newobj = (SHA3object *)PyObject_New(SHA3object, type); + SHA3object *newobj = PyObject_GC_New(SHA3object, type); if (newobj == NULL) { return NULL; } HASHLIB_INIT_MUTEX(newobj); + PyObject_GC_Track(newobj); return newobj; } @@ -166,15 +166,32 @@ py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) /* Internal methods for a hash object */ +static int +SHA3_clear(SHA3object *self) +{ + if (self->hash_state != NULL) { + Hacl_Hash_SHA3_free(self->hash_state); + self->hash_state = NULL; + } + return 0; +} + static void SHA3_dealloc(SHA3object *self) { - Hacl_Hash_SHA3_free(self->hash_state); PyTypeObject *tp = Py_TYPE(self); - PyObject_Free(self); + PyObject_GC_UnTrack(self); + (void)SHA3_clear(self); + tp->tp_free(self); Py_DECREF(tp); } +static int +SHA3_traverse(PyObject *self, visitproc visit, void *arg) +{ + Py_VISIT(Py_TYPE(self)); + return 0; +} /* External methods for a hash object */ @@ -335,6 +352,7 @@ static PyObject * SHA3_get_capacity_bits(SHA3object *self, void *closure) { uint32_t rate = Hacl_Hash_SHA3_block_len(self->hash_state) * 8; + assert(rate <= 1600); int capacity = 1600 - rate; return PyLong_FromLong(capacity); } @@ -366,12 +384,14 @@ static PyGetSetDef SHA3_getseters[] = { #define SHA3_TYPE_SLOTS(type_slots_obj, type_doc, type_methods, type_getseters) \ static PyType_Slot type_slots_obj[] = { \ + {Py_tp_clear, SHA3_clear}, \ {Py_tp_dealloc, SHA3_dealloc}, \ + {Py_tp_traverse, SHA3_traverse}, \ {Py_tp_doc, (char*)type_doc}, \ {Py_tp_methods, type_methods}, \ {Py_tp_getset, type_getseters}, \ {Py_tp_new, py_sha3_new}, \ - {0,0} \ + {0, NULL} \ } // Using _PyType_GetModuleState() on these types is safe since they @@ -380,7 +400,8 @@ static PyGetSetDef SHA3_getseters[] = { static PyType_Spec type_spec_obj = { \ .name = "_sha3." type_name, \ .basicsize = sizeof(SHA3object), \ - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE, \ + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE \ + | Py_TPFLAGS_HAVE_GC, \ .slots = type_slots \ } @@ -444,9 +465,7 @@ _SHAKE_digest(SHA3object *self, unsigned long digestlen, int hex) result = PyBytes_FromStringAndSize((const char *)digest, digestlen); } - if (digest != NULL) { - PyMem_Free(digest); - } + PyMem_Free(digest); return result; } @@ -563,7 +582,7 @@ _sha3_clear(PyObject *module) static void _sha3_free(void *module) { - _sha3_clear((PyObject *)module); + (void)_sha3_clear((PyObject *)module); } static int diff --git a/Objects/abstract.c b/Objects/abstract.c index f6647874d732f6..c92ef10aa79648 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1993,9 +1993,6 @@ PyObject * PySequence_Tuple(PyObject *v) { PyObject *it; /* iter(v) */ - Py_ssize_t n; /* guess for result tuple size */ - PyObject *result = NULL; - Py_ssize_t j; if (v == NULL) { return null_error(); @@ -2017,58 +2014,54 @@ PySequence_Tuple(PyObject *v) if (it == NULL) return NULL; - /* Guess result size and allocate space. */ - n = PyObject_LengthHint(v, 10); - if (n == -1) - goto Fail; - result = PyTuple_New(n); - if (result == NULL) - goto Fail; - - /* Fill the tuple. */ - for (j = 0; ; ++j) { + Py_ssize_t n; + PyObject *buffer[8]; + for (n = 0; n < 8; n++) { PyObject *item = PyIter_Next(it); if (item == NULL) { - if (PyErr_Occurred()) - goto Fail; - break; - } - if (j >= n) { - size_t newn = (size_t)n; - /* The over-allocation strategy can grow a bit faster - than for lists because unlike lists the - over-allocation isn't permanent -- we reclaim - the excess before the end of this routine. - So, grow by ten and then add 25%. - */ - newn += 10u; - newn += newn >> 2; - if (newn > PY_SSIZE_T_MAX) { - /* Check for overflow */ - PyErr_NoMemory(); - Py_DECREF(item); - goto Fail; + if (PyErr_Occurred()) { + goto fail; } - n = (Py_ssize_t)newn; - if (_PyTuple_Resize(&result, n) != 0) { - Py_DECREF(item); - goto Fail; + Py_DECREF(it); + return _PyTuple_FromArraySteal(buffer, n); + } + buffer[n] = item; + } + PyListObject *list = (PyListObject *)PyList_New(16); + if (list == NULL) { + goto fail; + } + assert(n == 8); + Py_SET_SIZE(list, n); + for (Py_ssize_t j = 0; j < n; j++) { + PyList_SET_ITEM(list, j, buffer[j]); + } + for (;;) { + PyObject *item = PyIter_Next(it); + if (item == NULL) { + if (PyErr_Occurred()) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } + break; + } + if (_PyList_AppendTakeRef(list, item) < 0) { + Py_DECREF(list); + Py_DECREF(it); + return NULL; } - PyTuple_SET_ITEM(result, j, item); } - - /* Cut tuple back if guess was too large. */ - if (j < n && - _PyTuple_Resize(&result, j) != 0) - goto Fail; - Py_DECREF(it); - return result; - -Fail: - Py_XDECREF(result); + PyObject *res = _PyList_AsTupleAndClear(list); + Py_DECREF(list); + return res; +fail: Py_DECREF(it); + while (n > 0) { + n--; + Py_DECREF(buffer[n]); + } return NULL; } diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index 8c7651f0f3aa45..533089d25cd73a 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -2543,7 +2543,12 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) bot = _PyLong_DigitValue[*str]; if (bot >= 16) { - invalid_char = str - PyUnicode_1BYTE_DATA(string); + /* Check if we had a second digit */ + if (str >= end){ + invalid_char = -1; + } else { + invalid_char = str - PyUnicode_1BYTE_DATA(string); + } goto error; } str++; @@ -2554,9 +2559,14 @@ _PyBytes_FromHex(PyObject *string, int use_bytearray) return _PyBytesWriter_Finish(&writer, buf); error: - PyErr_Format(PyExc_ValueError, - "non-hexadecimal number found in " - "fromhex() arg at position %zd", invalid_char); + if (invalid_char == -1) { + PyErr_SetString(PyExc_ValueError, + "fromhex() arg must contain an even number of hexadecimal digits"); + } else { + PyErr_Format(PyExc_ValueError, + "non-hexadecimal number found in " + "fromhex() arg at position %zd", invalid_char); + } _PyBytesWriter_Dealloc(&writer); return NULL; } diff --git a/Objects/cellobject.c b/Objects/cellobject.c index 590c8a80857699..ec2eeb1a855b63 100644 --- a/Objects/cellobject.c +++ b/Objects/cellobject.c @@ -82,6 +82,17 @@ cell_dealloc(PyObject *self) PyObject_GC_Del(op); } +static PyObject * +cell_compare_impl(PyObject *a, PyObject *b, int op) +{ + if (a != NULL && b != NULL) { + return PyObject_RichCompare(a, b, op); + } + else { + Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + } +} + static PyObject * cell_richcompare(PyObject *a, PyObject *b, int op) { @@ -92,27 +103,28 @@ cell_richcompare(PyObject *a, PyObject *b, int op) if (!PyCell_Check(a) || !PyCell_Check(b)) { Py_RETURN_NOTIMPLEMENTED; } + PyObject *a_ref = PyCell_GetRef((PyCellObject *)a); + PyObject *b_ref = PyCell_GetRef((PyCellObject *)b); /* compare cells by contents; empty cells come before anything else */ - a = ((PyCellObject *)a)->ob_ref; - b = ((PyCellObject *)b)->ob_ref; - if (a != NULL && b != NULL) - return PyObject_RichCompare(a, b, op); + PyObject *res = cell_compare_impl(a_ref, b_ref, op); - Py_RETURN_RICHCOMPARE(b == NULL, a == NULL, op); + Py_XDECREF(a_ref); + Py_XDECREF(b_ref); + return res; } static PyObject * cell_repr(PyObject *self) { - PyCellObject *op = _PyCell_CAST(self); - if (op->ob_ref == NULL) { - return PyUnicode_FromFormat("", op); + PyObject *ref = PyCell_GetRef((PyCellObject *)self); + if (ref == NULL) { + return PyUnicode_FromFormat("", self); } - - return PyUnicode_FromFormat("", - op, Py_TYPE(op->ob_ref)->tp_name, - op->ob_ref); + PyObject *res = PyUnicode_FromFormat("", + self, Py_TYPE(ref)->tp_name, ref); + Py_DECREF(ref); + return res; } static int @@ -135,18 +147,20 @@ static PyObject * cell_get_contents(PyObject *self, void *closure) { PyCellObject *op = _PyCell_CAST(self); - if (op->ob_ref == NULL) { + PyObject *res = PyCell_GetRef(op); + if (res == NULL) { PyErr_SetString(PyExc_ValueError, "Cell is empty"); return NULL; } - return Py_NewRef(op->ob_ref); + return res; } static int cell_set_contents(PyObject *self, PyObject *obj, void *Py_UNUSED(ignored)) { - PyCellObject *op = _PyCell_CAST(self); - Py_XSETREF(op->ob_ref, Py_XNewRef(obj)); + PyCellObject *cell = _PyCell_CAST(self); + Py_XINCREF(obj); + PyCell_SetTakeRef((PyCellObject *)cell, obj); return 0; } diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 185d7819b6b84a..a6cf1f431a15b0 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -418,4 +418,59 @@ memoryview_hex(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=0a93f08110630633 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(memoryview_count__doc__, +"count($self, value, /)\n" +"--\n" +"\n" +"Count the number of occurrences of a value."); + +#define MEMORYVIEW_COUNT_METHODDEF \ + {"count", (PyCFunction)memoryview_count, METH_O, memoryview_count__doc__}, + +PyDoc_STRVAR(memoryview_index__doc__, +"index($self, value, start=0, stop=sys.maxsize, /)\n" +"--\n" +"\n" +"Return the index of the first occurrence of a value.\n" +"\n" +"Raises ValueError if the value is not present."); + +#define MEMORYVIEW_INDEX_METHODDEF \ + {"index", _PyCFunction_CAST(memoryview_index), METH_FASTCALL, memoryview_index__doc__}, + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop); + +static PyObject * +memoryview_index(PyMemoryViewObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *value; + Py_ssize_t start = 0; + Py_ssize_t stop = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { + goto exit; + } + value = args[0]; + if (nargs < 2) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[1], &start)) { + goto exit; + } + if (nargs < 3) { + goto skip_optional; + } + if (!_PyEval_SliceIndexNotNone(args[2], &stop)) { + goto exit; + } +skip_optional: + return_value = memoryview_index_impl(self, value, start, stop); + +exit: + return return_value; +} +/*[clinic end generated code: output=132893ef5f67ad73 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 148350cc4b9195..eb8de136ee6432 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -1867,14 +1867,11 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyCodeObject *co) { - assert(Py_REFCNT(co) == 0); - Py_SET_REFCNT(co, 1); + _PyObject_ResurrectStart((PyObject *)co); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); - if (Py_REFCNT(co) > 1) { - Py_SET_REFCNT(co, Py_REFCNT(co) - 1); + if (_PyObject_ResurrectEnd((PyObject *)co)) { return; } - Py_SET_REFCNT(co, 0); #ifdef Py_GIL_DISABLED PyObject_GC_UnTrack(co); diff --git a/Objects/complexobject.c b/Objects/complexobject.c index 8fbca3cb02d80a..bf6187efac941f 100644 --- a/Objects/complexobject.c +++ b/Objects/complexobject.c @@ -85,11 +85,63 @@ _Py_c_neg(Py_complex a) } Py_complex -_Py_c_prod(Py_complex a, Py_complex b) +_Py_c_prod(Py_complex z, Py_complex w) { - Py_complex r; - r.real = a.real*b.real - a.imag*b.imag; - r.imag = a.real*b.imag + a.imag*b.real; + double a = z.real, b = z.imag, c = w.real, d = w.imag; + double ac = a*c, bd = b*d, ad = a*d, bc = b*c; + Py_complex r = {ac - bd, ad + bc}; + + /* Recover infinities that computed as nan+nanj. See e.g. the C11, + Annex G.5.1, routine _Cmultd(). */ + if (isnan(r.real) && isnan(r.imag)) { + int recalc = 0; + + if (isinf(a) || isinf(b)) { /* z is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + a = copysign(isinf(a) ? 1.0 : 0.0, a); + b = copysign(isinf(b) ? 1.0 : 0.0, b); + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (isinf(c) || isinf(d)) { /* w is infinite */ + /* "Box" the infinity and change nans in the other factor to 0 */ + c = copysign(isinf(c) ? 1.0 : 0.0, c); + d = copysign(isinf(d) ? 1.0 : 0.0, d); + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + recalc = 1; + } + if (!recalc && (isinf(ac) || isinf(bd) || isinf(ad) || isinf(bc))) { + /* Recover infinities from overflow by changing nans to 0 */ + if (isnan(a)) { + a = copysign(0.0, a); + } + if (isnan(b)) { + b = copysign(0.0, b); + } + if (isnan(c)) { + c = copysign(0.0, c); + } + if (isnan(d)) { + d = copysign(0.0, d); + } + recalc = 1; + } + if (recalc) { + r.real = Py_INFINITY*(a*c - b*d); + r.imag = Py_INFINITY*(a*d + b*c); + } + } + return r; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 49b213eaa817e2..05c93a3e448181 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -588,11 +588,14 @@ estimate_log2_keysize(Py_ssize_t n) /* This immutable, empty PyDictKeysObject is used for PyDict_Clear() * (which cannot fail and thus can do no allocation). + * + * See https://github.com/python/cpython/pull/127568#discussion_r1868070614 + * for the rationale of using dk_log2_index_bytes=3 instead of 0. */ static PyDictKeysObject empty_keys_struct = { _Py_DICT_IMMORTAL_INITIAL_REFCNT, /* dk_refcnt */ 0, /* dk_log2_size */ - 0, /* dk_log2_index_bytes */ + 3, /* dk_log2_index_bytes */ DICT_KEYS_UNICODE, /* dk_kind */ #ifdef Py_GIL_DISABLED {0}, /* dk_mutex */ @@ -3162,14 +3165,11 @@ dict_dealloc(PyObject *self) { PyDictObject *mp = (PyDictObject *)self; PyInterpreterState *interp = _PyInterpreterState_GET(); - assert(Py_REFCNT(mp) == 0); - Py_SET_REFCNT(mp, 1); + _PyObject_ResurrectStart(self); _PyDict_NotifyEvent(interp, PyDict_EVENT_DEALLOCATED, mp, NULL, NULL); - if (Py_REFCNT(mp) > 1) { - Py_SET_REFCNT(mp, Py_REFCNT(mp) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(mp, 0); PyDictValues *values = mp->ma_values; PyDictKeysObject *keys = mp->ma_keys; Py_ssize_t i, n; @@ -7300,7 +7300,7 @@ _PyDict_DetachFromObject(PyDictObject *mp, PyObject *obj) // We could be called with an unlocked dict when the caller knows the // values are already detached, so we assert after inline values check. - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + ASSERT_WORLD_STOPPED_OR_OBJ_LOCKED(mp); assert(mp->ma_values->embedded == 1); assert(mp->ma_values->valid == 1); assert(Py_TYPE(obj)->tp_flags & Py_TPFLAGS_INLINE_VALUES); diff --git a/Objects/exceptions.c b/Objects/exceptions.c index 6fbe0f197eaebf..287cbc25305964 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -2708,6 +2708,46 @@ set_unicodefromstring(PyObject **attr, const char *value) return 0; } +/* + * Adjust the (inclusive) 'start' value of a UnicodeError object. + * + * The 'start' can be negative or not, but when adjusting the value, + * we clip it in [0, max(0, objlen - 1)] and do not interpret it as + * a relative offset. + */ +static inline Py_ssize_t +unicode_error_adjust_start(Py_ssize_t start, Py_ssize_t objlen) +{ + assert(objlen >= 0); + if (start < 0) { + start = 0; + } + if (start >= objlen) { + start = objlen == 0 ? 0 : objlen - 1; + } + return start; +} + +/* + * Adjust the (exclusive) 'end' value of a UnicodeError object. + * + * The 'end' can be negative or not, but when adjusting the value, + * we clip it in [min(1, objlen), max(min(1, objlen), objlen)] and + * do not interpret it as a relative offset. + */ +static inline Py_ssize_t +unicode_error_adjust_end(Py_ssize_t end, Py_ssize_t objlen) +{ + assert(objlen >= 0); + if (end < 1) { + end = 1; + } + if (end > objlen) { + end = objlen; + } + return end; +} + PyObject * PyUnicodeEncodeError_GetEncoding(PyObject *exc) { @@ -2739,38 +2779,31 @@ PyUnicodeTranslateError_GetObject(PyObject *exc) } int -PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeEncodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { return -1; - *start = ((PyUnicodeErrorObject *)exc)->start; - size = PyUnicode_GET_LENGTH(obj); - if (*start<0) - *start = 0; /*XXX check for values <0*/ - if (*start>=size) - *start = size-1; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); Py_DECREF(obj); + *start = unicode_error_adjust_start(exc->start, size); return 0; } int -PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) +PyUnicodeDecodeError_GetStart(PyObject *self, Py_ssize_t *start) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_string(exc->object, "object"); + if (obj == NULL) { return -1; - size = PyBytes_GET_SIZE(obj); - *start = ((PyUnicodeErrorObject *)exc)->start; - if (*start<0) - *start = 0; - if (*start>=size) - *start = size-1; + } + Py_ssize_t size = PyBytes_GET_SIZE(obj); Py_DECREF(obj); + *start = unicode_error_adjust_start(exc->start, size); return 0; } @@ -2782,63 +2815,61 @@ PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) } +static inline int +unicode_error_set_start_impl(PyObject *self, Py_ssize_t start) +{ + ((PyUnicodeErrorObject *)self)->start = start; + return 0; +} + + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) { - ((PyUnicodeErrorObject *)exc)->start = start; - return 0; + return unicode_error_set_start_impl(exc, start); } int -PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeEncodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_unicode(((PyUnicodeErrorObject *)exc)->object, - "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_unicode(exc->object, "object"); + if (obj == NULL) { return -1; - *end = ((PyUnicodeErrorObject *)exc)->end; - size = PyUnicode_GET_LENGTH(obj); - if (*end<1) - *end = 1; - if (*end>size) - *end = size; + } + Py_ssize_t size = PyUnicode_GET_LENGTH(obj); Py_DECREF(obj); + *end = unicode_error_adjust_end(exc->end, size); return 0; } int -PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) +PyUnicodeDecodeError_GetEnd(PyObject *self, Py_ssize_t *end) { - Py_ssize_t size; - PyObject *obj = get_string(((PyUnicodeErrorObject *)exc)->object, "object"); - if (!obj) + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + PyObject *obj = get_string(exc->object, "object"); + if (obj == NULL) { return -1; - size = PyBytes_GET_SIZE(obj); - *end = ((PyUnicodeErrorObject *)exc)->end; - if (*end<1) - *end = 1; - if (*end>size) - *end = size; + } + Py_ssize_t size = PyBytes_GET_SIZE(obj); Py_DECREF(obj); + *end = unicode_error_adjust_end(exc->end, size); return 0; } @@ -2850,27 +2881,32 @@ PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) } -int -PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +static inline int +unicode_error_set_end_impl(PyObject *exc, Py_ssize_t end) { ((PyUnicodeErrorObject *)exc)->end = end; return 0; } +int +PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) +{ + return unicode_error_set_end_impl(exc, end); +} + + int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + return unicode_error_set_end_impl(exc, end); } int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) { - ((PyUnicodeErrorObject *)exc)->end = end; - return 0; + return unicode_error_set_end_impl(exc, end); } PyObject * @@ -2966,28 +3002,25 @@ static PyMemberDef UnicodeError_members[] = { static int UnicodeEncodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *err; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - err = (PyUnicodeErrorObject *)self; - - Py_CLEAR(err->encoding); - Py_CLEAR(err->object); - Py_CLEAR(err->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UUnnU", - &err->encoding, &err->object, - &err->start, &err->end, &err->reason)) { - err->encoding = err->object = err->reason = NULL; + &encoding, &object, &start, &end, &reason)) + { return -1; } - Py_INCREF(err->encoding); - Py_INCREF(err->object); - Py_INCREF(err->reason); - + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } @@ -3073,44 +3106,42 @@ PyObject *PyExc_UnicodeEncodeError = (PyObject *)&_PyExc_UnicodeEncodeError; static int UnicodeDecodeError_init(PyObject *self, PyObject *args, PyObject *kwds) { - PyUnicodeErrorObject *ude; - - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - ude = (PyUnicodeErrorObject *)self; - - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); + PyObject *encoding = NULL, *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; if (!PyArg_ParseTuple(args, "UOnnU", - &ude->encoding, &ude->object, - &ude->start, &ude->end, &ude->reason)) { - ude->encoding = ude->object = ude->reason = NULL; - return -1; + &encoding, &object, &start, &end, &reason)) + { + return -1; } - Py_INCREF(ude->encoding); - Py_INCREF(ude->object); - Py_INCREF(ude->reason); - - if (!PyBytes_Check(ude->object)) { + if (PyBytes_Check(object)) { + Py_INCREF(object); // make 'object' a strong reference + } + else { Py_buffer view; - if (PyObject_GetBuffer(ude->object, &view, PyBUF_SIMPLE) != 0) - goto error; - Py_XSETREF(ude->object, PyBytes_FromStringAndSize(view.buf, view.len)); + if (PyObject_GetBuffer(object, &view, PyBUF_SIMPLE) != 0) { + return -1; + } + // 'object' is borrowed, so we can re-use the variable + object = PyBytes_FromStringAndSize(view.buf, view.len); PyBuffer_Release(&view); - if (!ude->object) - goto error; + if (object == NULL) { + return -1; + } } - return 0; -error: - Py_CLEAR(ude->encoding); - Py_CLEAR(ude->object); - Py_CLEAR(ude->reason); - return -1; + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->encoding, Py_NewRef(encoding)); + Py_XSETREF(exc->object, object /* already a strong reference */); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); + return 0; } static PyObject * @@ -3192,25 +3223,24 @@ PyUnicodeDecodeError_Create( */ static int -UnicodeTranslateError_init(PyUnicodeErrorObject *self, PyObject *args, - PyObject *kwds) +UnicodeTranslateError_init(PyObject *self, PyObject *args, PyObject *kwds) { - if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) + if (BaseException_init((PyBaseExceptionObject *)self, args, kwds) == -1) { return -1; + } - Py_CLEAR(self->object); - Py_CLEAR(self->reason); + PyObject *object = NULL, *reason = NULL; // borrowed + Py_ssize_t start = -1, end = -1; - if (!PyArg_ParseTuple(args, "UnnU", - &self->object, - &self->start, &self->end, &self->reason)) { - self->object = self->reason = NULL; + if (!PyArg_ParseTuple(args, "UnnU", &object, &start, &end, &reason)) { return -1; } - Py_INCREF(self->object); - Py_INCREF(self->reason); - + PyUnicodeErrorObject *exc = (PyUnicodeErrorObject *)self; + Py_XSETREF(exc->object, Py_NewRef(object)); + exc->start = start; + exc->end = end; + Py_XSETREF(exc->reason, Py_NewRef(reason)); return 0; } diff --git a/Objects/frameobject.c b/Objects/frameobject.c index c743c254848d3a..03ed2b9480f8c9 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -8,6 +8,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetDict() #include "pycore_modsupport.h" // _PyArg_CheckPositional() #include "pycore_object.h" // _PyObject_GC_UNTRACK() +#include "pycore_cell.h" // PyCell_GetRef() PyCell_SetTakeRef() #include "pycore_opcode_metadata.h" // _PyOpcode_Deopt, _PyOpcode_Caches @@ -187,11 +188,8 @@ framelocalsproxy_setitem(PyObject *self, PyObject *key, PyObject *value) } } if (cell != NULL) { - PyObject *oldvalue_o = PyCell_GET(cell); - if (value != oldvalue_o) { - PyCell_SET(cell, Py_XNewRef(value)); - Py_XDECREF(oldvalue_o); - } + Py_XINCREF(value); + PyCell_SetTakeRef((PyCellObject *)cell, value); } else if (value != PyStackRef_AsPyObjectBorrow(oldvalue)) { PyStackRef_XCLOSE(fast[i]); fast[i] = PyStackRef_FromPyObjectNew(value); @@ -1987,19 +1985,25 @@ frame_get_var(_PyInterpreterFrame *frame, PyCodeObject *co, int i, if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); - value = PyCell_GET(value); + value = PyCell_GetRef((PyCellObject *)value); } else if (kind & CO_FAST_CELL) { if (value != NULL) { if (PyCell_Check(value)) { assert(!_PyFrame_IsIncomplete(frame)); - value = PyCell_GET(value); + value = PyCell_GetRef((PyCellObject *)value); + } + else { + // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), + // with the initial value set when the frame was created... + // (unlikely) ...or it was set via the f_locals proxy. + Py_INCREF(value); } - // (likely) Otherwise it is an arg (kind & CO_FAST_LOCAL), - // with the initial value set when the frame was created... - // (unlikely) ...or it was set via the f_locals proxy. } } + else { + Py_XINCREF(value); + } } *pvalue = value; return 1; @@ -2076,14 +2080,14 @@ PyFrame_GetVar(PyFrameObject *frame_obj, PyObject *name) continue; } - PyObject *value; // borrowed reference + PyObject *value; if (!frame_get_var(frame, co, i, &value)) { break; } if (value == NULL) { break; } - return Py_NewRef(value); + return value; } PyErr_Format(PyExc_NameError, "variable %R does not exist", name); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 4ba47285f7152f..cca7f01498013e 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1092,14 +1092,11 @@ static void func_dealloc(PyObject *self) { PyFunctionObject *op = _PyFunction_CAST(self); - assert(Py_REFCNT(op) == 0); - Py_SET_REFCNT(op, 1); + _PyObject_ResurrectStart(self); handle_func_event(PyFunction_EVENT_DESTROY, op, NULL); - if (Py_REFCNT(op) > 1) { - Py_SET_REFCNT(op, Py_REFCNT(op) - 1); + if (_PyObject_ResurrectEnd(self)) { return; } - Py_SET_REFCNT(op, 0); _PyObject_GC_UNTRACK(op); if (op->func_weakreflist != NULL) { PyObject_ClearWeakRefs((PyObject *) op); diff --git a/Objects/listobject.c b/Objects/listobject.c index bb0040cbe9f272..a877bad66be45f 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_ceval.h" // _PyEval_GetBuiltin() +#include "pycore_critical_section.h" // _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED() #include "pycore_dict.h" // _PyDictViewObject #include "pycore_freelist.h" // _Py_FREELIST_FREE(), _Py_FREELIST_POP() #include "pycore_pyatomic_ft_wrappers.h" @@ -68,6 +69,25 @@ free_list_items(PyObject** items, bool use_qsbr) #endif } +static void +ensure_shared_on_resize(PyListObject *self) +{ +#ifdef Py_GIL_DISABLED + // We can't use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here because + // the `CALL_LIST_APPEND` bytecode handler may lock the list without + // a critical section. + assert(Py_REFCNT(self) == 1 || PyMutex_IsLocked(&_PyObject_CAST(self)->ob_mutex)); + + // Ensure that the list array is freed using QSBR if we are not the + // owning thread. + if (!_Py_IsOwnedByCurrentThread((PyObject *)self) && + !_PyObject_GC_IS_SHARED(self)) + { + _PyObject_GC_SET_SHARED(self); + } +#endif +} + /* Ensure ob_item has room for at least newsize elements, and set * ob_size to newsize. If newsize > ob_size on entry, the content * of the new slots at exit is undefined heap trash; it's the caller's @@ -117,6 +137,8 @@ list_resize(PyListObject *self, Py_ssize_t newsize) if (newsize == 0) new_allocated = 0; + ensure_shared_on_resize(self); + #ifdef Py_GIL_DISABLED _PyListArray *array = list_allocate_array(new_allocated); if (array == NULL) { @@ -391,6 +413,12 @@ PyList_GetItemRef(PyObject *op, Py_ssize_t i) return item; } +PyObject * +_PyList_GetItemRef(PyListObject *list, Py_ssize_t i) +{ + return list_get_item_ref(list, i); +} + int PyList_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) @@ -798,6 +826,9 @@ list_clear_impl(PyListObject *a, bool is_resize) Py_XDECREF(items[i]); } #ifdef Py_GIL_DISABLED + if (is_resize) { + ensure_shared_on_resize(a); + } bool use_qsbr = is_resize && _PyObject_GC_IS_SHARED(a); #else bool use_qsbr = false; @@ -932,10 +963,12 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) Py_ssize_t n = PyList_GET_SIZE(a); PyObject *copy = list_slice_lock_held(a, 0, n); if (copy == NULL) { - return -1; + ret = -1; + } + else { + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); } - ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); - Py_DECREF(copy); Py_END_CRITICAL_SECTION(); } else if (v != NULL && PyList_CheckExact(v)) { @@ -1412,7 +1445,9 @@ PyList_Clear(PyObject *self) PyErr_BadInternalCall(); return -1; } + Py_BEGIN_CRITICAL_SECTION(self); list_clear((PyListObject*)self); + Py_END_CRITICAL_SECTION(); return 0; } @@ -3063,6 +3098,7 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) Py_XDECREF(final_ob_item[i]); } #ifdef Py_GIL_DISABLED + ensure_shared_on_resize(self); bool use_qsbr = _PyObject_GC_IS_SHARED(self); #else bool use_qsbr = false; @@ -3138,6 +3174,25 @@ PyList_AsTuple(PyObject *v) return ret; } +PyObject * +_PyList_AsTupleAndClear(PyListObject *self) +{ + assert(self != NULL); + PyObject *ret; + if (self->ob_item == NULL) { + return PyTuple_New(0); + } + Py_BEGIN_CRITICAL_SECTION(self); + PyObject **items = self->ob_item; + Py_ssize_t size = Py_SIZE(self); + self->ob_item = NULL; + Py_SET_SIZE(self, 0); + ret = _PyTuple_FromArraySteal(items, size); + free_list_items(items, false); + Py_END_CRITICAL_SECTION(); + return ret; +} + PyObject * _PyList_FromStackRefSteal(const _PyStackRef *src, Py_ssize_t n) { @@ -3384,7 +3439,9 @@ list___init___impl(PyListObject *self, PyObject *iterable) /* Empty previous contents */ if (self->ob_item != NULL) { + Py_BEGIN_CRITICAL_SECTION(self); list_clear(self); + Py_END_CRITICAL_SECTION(); } if (iterable != NULL) { if (_list_extend(self, iterable) < 0) { @@ -3557,8 +3614,10 @@ adjust_slice_indexes(PyListObject *lst, } static int -list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) +list_ass_subscript_lock_held(PyObject *_self, PyObject *item, PyObject *value) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(_self); + PyListObject *self = (PyListObject *)_self; if (_PyIndex_Check(item)) { Py_ssize_t i = PyNumber_AsSsize_t(item, PyExc_IndexError); @@ -3566,7 +3625,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) return -1; if (i < 0) i += PyList_GET_SIZE(self); - return list_ass_item((PyObject *)self, i, value); + return list_ass_item_lock_held(self, i, value); } else if (PySlice_Check(item)) { Py_ssize_t start, stop, step; @@ -3586,7 +3645,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) step); if (step == 1) - return list_ass_slice(self, start, stop, value); + return list_ass_slice_lock_held(self, start, stop, value); if (slicelength <= 0) return 0; @@ -3652,10 +3711,8 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - Py_BEGIN_CRITICAL_SECTION(value); - seq = list_slice_lock_held((PyListObject*)value, 0, + seq = list_slice_lock_held((PyListObject *)value, 0, Py_SIZE(value)); - Py_END_CRITICAL_SECTION(); } else { seq = PySequence_Fast(value, @@ -3669,7 +3726,7 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) step); if (step == 1) { - int res = list_ass_slice(self, start, stop, seq); + int res = list_ass_slice_lock_held(self, start, stop, seq); Py_DECREF(seq); return res; } @@ -3725,6 +3782,24 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) } } +static int +list_ass_subscript(PyObject *self, PyObject *item, PyObject *value) +{ + int res; +#ifdef Py_GIL_DISABLED + if (PySlice_Check(item) && value != NULL && PyList_CheckExact(value)) { + Py_BEGIN_CRITICAL_SECTION2(self, value); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION2(); + return res; + } +#endif + Py_BEGIN_CRITICAL_SECTION(self); + res = list_ass_subscript_lock_held(self, item, value); + Py_END_CRITICAL_SECTION(); + return res; +} + static PyMappingMethods list_as_mapping = { list_length, list_subscript, diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index 25634f997ac66b..afe2dcb127adb4 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -2748,6 +2748,141 @@ static PySequenceMethods memory_as_sequence = { }; +/****************************************************************************/ +/* Counting */ +/****************************************************************************/ + +/*[clinic input] +memoryview.count + + value: object + / + +Count the number of occurrences of a value. +[clinic start generated code]*/ + +static PyObject * +memoryview_count(PyMemoryViewObject *self, PyObject *value) +/*[clinic end generated code: output=e2c255a8d54eaa12 input=e3036ce1ed7d1823]*/ +{ + PyObject *iter = PyObject_GetIter(_PyObject_CAST(self)); + if (iter == NULL) { + return NULL; + } + + Py_ssize_t count = 0; + PyObject *item = NULL; + while (PyIter_NextItem(iter, &item)) { + if (item == NULL) { + Py_DECREF(iter); + return NULL; + } + if (item == value) { + Py_DECREF(item); + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + continue; + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + count++; // no overflow since count <= len(mv) <= PY_SSIZE_T_MAX + } + else if (contained < 0) { + Py_DECREF(iter); + return NULL; + } + } + Py_DECREF(iter); + return PyLong_FromSsize_t(count); +} + + +/**************************************************************************/ +/* Lookup */ +/**************************************************************************/ + +/*[clinic input] +memoryview.index + + value: object + start: slice_index(accept={int}) = 0 + stop: slice_index(accept={int}, c_default="PY_SSIZE_T_MAX") = sys.maxsize + / + +Return the index of the first occurrence of a value. + +Raises ValueError if the value is not present. +[clinic start generated code]*/ + +static PyObject * +memoryview_index_impl(PyMemoryViewObject *self, PyObject *value, + Py_ssize_t start, Py_ssize_t stop) +/*[clinic end generated code: output=e0185e3819e549df input=0697a0165bf90b5a]*/ +{ + const Py_buffer *view = &self->view; + CHECK_RELEASED(self); + + if (view->ndim == 0) { + PyErr_SetString(PyExc_TypeError, "invalid lookup on 0-dim memory"); + return NULL; + } + + if (view->ndim == 1) { + Py_ssize_t n = view->shape[0]; + + if (start < 0) { + start = Py_MAX(start + n, 0); + } + + if (stop < 0) { + stop = Py_MAX(stop + n, 0); + } + + stop = Py_MIN(stop, n); + assert(stop >= 0); + assert(stop <= n); + + start = Py_MIN(start, stop); + assert(0 <= start); + assert(start <= stop); + + PyObject *obj = _PyObject_CAST(self); + for (Py_ssize_t index = start; index < stop; index++) { + // Note: while memoryviews can be mutated during iterations + // when calling the == operator, their shape cannot. As such, + // it is safe to assume that the index remains valid for the + // entire loop. + assert(index < n); + + PyObject *item = memory_item(obj, index); + if (item == NULL) { + return NULL; + } + if (item == value) { + Py_DECREF(item); + return PyLong_FromSsize_t(index); + } + int contained = PyObject_RichCompareBool(item, value, Py_EQ); + Py_DECREF(item); + if (contained > 0) { // more likely than 'contained < 0' + return PyLong_FromSsize_t(index); + } + else if (contained < 0) { + return NULL; + } + } + + PyErr_SetString(PyExc_ValueError, "memoryview.index(x): x not found"); + return NULL; + } + + PyErr_SetString(PyExc_NotImplementedError, + "multi-dimensional lookup is not implemented"); + return NULL; + +} + + /**************************************************************************/ /* Comparisons */ /**************************************************************************/ @@ -3284,6 +3419,8 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_CAST_METHODDEF MEMORYVIEW_TOREADONLY_METHODDEF MEMORYVIEW__FROM_FLAGS_METHODDEF + MEMORYVIEW_COUNT_METHODDEF + MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, diff --git a/Objects/object.c b/Objects/object.c index 8868fa29066404..c64675b5e1d6c2 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -362,8 +362,10 @@ is_dead(PyObject *o) } # endif -void -_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +// Decrement the shared reference count of an object. Return 1 if the object +// is dead and should be deallocated, 0 otherwise. +static int +_Py_DecRefSharedIsDead(PyObject *o, const char *filename, int lineno) { // Should we queue the object for the owning thread to merge? int should_queue; @@ -404,6 +406,15 @@ _Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) } else if (new_shared == _Py_REF_MERGED) { // refcount is zero AND merged + return 1; + } + return 0; +} + +void +_Py_DecRefSharedDebug(PyObject *o, const char *filename, int lineno) +{ + if (_Py_DecRefSharedIsDead(o, filename, lineno)) { _Py_Dealloc(o); } } @@ -472,6 +483,26 @@ _Py_ExplicitMergeRefcount(PyObject *op, Py_ssize_t extra) &shared, new_shared)); return refcnt; } + +// The more complicated "slow" path for undoing the resurrection of an object. +int +_PyObject_ResurrectEndSlow(PyObject *op) +{ + if (_Py_IsImmortal(op)) { + return 1; + } + if (_Py_IsOwnedByCurrentThread(op)) { + // If the object is owned by the current thread, give up ownership and + // merge the refcount. This isn't necessary in all cases, but it + // simplifies the implementation. + Py_ssize_t refcount = _Py_ExplicitMergeRefcount(op, -1); + return refcount != 0; + } + int is_dead = _Py_DecRefSharedIsDead(op, NULL, 0); + return !is_dead; +} + + #endif /* Py_GIL_DISABLED */ @@ -550,7 +581,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) } /* Temporarily resurrect the object. */ - Py_SET_REFCNT(self, 1); + _PyObject_ResurrectStart(self); PyObject_CallFinalizer(self); @@ -560,8 +591,7 @@ PyObject_CallFinalizerFromDealloc(PyObject *self) /* Undo the temporary resurrection; can't use DECREF here, it would * cause a recursive call. */ - Py_SET_REFCNT(self, Py_REFCNT(self) - 1); - if (Py_REFCNT(self) == 0) { + if (!_PyObject_ResurrectEnd(self)) { return 0; /* this is the normal path out */ } @@ -2445,10 +2475,16 @@ new_reference(PyObject *op) { // Skip the immortal object check in Py_SET_REFCNT; always set refcnt to 1 #if !defined(Py_GIL_DISABLED) +#if SIZEOF_VOID_P > 4 + op->ob_refcnt_full = 1; + assert(op->ob_refcnt == 1); + assert(op->ob_flags == 0); +#else op->ob_refcnt = 1; +#endif #else op->ob_tid = _Py_ThreadId(); - op->_padding = 0; + op->ob_flags = 0; op->ob_mutex = (PyMutex){ 0 }; op->ob_gc_bits = 0; op->ob_ref_local = 1; @@ -2485,6 +2521,10 @@ _Py_SetImmortalUntracked(PyObject *op) || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); } #endif + // Check if already immortal to avoid degrading from static immortal to plain immortal + if (_Py_IsImmortal(op)) { + return; + } #ifdef Py_GIL_DISABLED op->ob_tid = _Py_UNOWNED_TID; op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 2cc0377f68f990..b103deb01ca712 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2910,9 +2910,16 @@ static inline void _PyMem_DebugCheckGIL(const char *func) { if (!PyGILState_Check()) { +#ifndef Py_GIL_DISABLED _Py_FatalErrorFunc(func, "Python memory allocator called " "without holding the GIL"); +#else + _Py_FatalErrorFunc(func, + "Python memory allocator called " + "without an active thread state. " + "Are you trying to call it inside of a Py_BEGIN_ALLOW_THREADS block?"); +#endif } } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2611404a3d0d61..2068d6aa9be52b 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -19,6 +19,7 @@ #include "pycore_typeobject.h" // struct type_cache #include "pycore_unionobject.h" // _Py_union_type_or #include "pycore_weakref.h" // _PyWeakref_GET_REF() +#include "pycore_cell.h" // PyCell_GetRef() #include "opcode.h" // MAKE_CELL #include // ptrdiff_t @@ -5527,9 +5528,12 @@ _PyTypes_AfterFork(void) } /* Internal API to look for a name through the MRO. - This returns a borrowed reference, and doesn't set an exception! */ + This returns a strong reference, and doesn't set an exception! + If nonzero, version is set to the value of type->tp_version at the time of + the lookup. +*/ PyObject * -_PyType_LookupRef(PyTypeObject *type, PyObject *name) +_PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *version) { PyObject *res; int error; @@ -5552,6 +5556,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) // If the sequence is still valid then we're done if (value == NULL || _Py_TryIncref(value)) { if (_PySeqLock_EndRead(&entry->sequence, sequence)) { + if (version != NULL) { + *version = entry_version; + } return value; } Py_XDECREF(value); @@ -5573,6 +5580,9 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) OBJECT_STAT_INC_COND(type_cache_hits, !is_dunder_name(name)); OBJECT_STAT_INC_COND(type_cache_dunder_hits, is_dunder_name(name)); Py_XINCREF(entry->value); + if (version != NULL) { + *version = entry->version; + } return entry->value; } #endif @@ -5586,12 +5596,12 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) // anyone else can modify our mro or mutate the type. int has_version = 0; - int version = 0; + unsigned int assigned_version = 0; BEGIN_TYPE_LOCK(); res = find_name_in_mro(type, name, &error); if (MCACHE_CACHEABLE_NAME(name)) { has_version = assign_version_tag(interp, type); - version = type->tp_version_tag; + assigned_version = type->tp_version_tag; } END_TYPE_LOCK(); @@ -5608,28 +5618,67 @@ _PyType_LookupRef(PyTypeObject *type, PyObject *name) if (error == -1) { PyErr_Clear(); } + if (version != NULL) { + // 0 is not a valid version + *version = 0; + } return NULL; } if (has_version) { #if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, version, res); + update_cache_gil_disabled(entry, name, assigned_version, res); #else - PyObject *old_value = update_cache(entry, name, version, res); + PyObject *old_value = update_cache(entry, name, assigned_version, res); Py_DECREF(old_value); #endif } + if (version != NULL) { + // 0 is not a valid version + *version = has_version ? assigned_version : 0; + } return res; } +/* Internal API to look for a name through the MRO. + This returns a strong reference, and doesn't set an exception! +*/ +PyObject * +_PyType_LookupRef(PyTypeObject *type, PyObject *name) +{ + return _PyType_LookupRefAndVersion(type, name, NULL); +} + +/* Internal API to look for a name through the MRO. + This returns a borrowed reference, and doesn't set an exception! */ PyObject * _PyType_Lookup(PyTypeObject *type, PyObject *name) { - PyObject *res = _PyType_LookupRef(type, name); + PyObject *res = _PyType_LookupRefAndVersion(type, name, NULL); Py_XDECREF(res); return res; } +int +_PyType_CacheInitForSpecialization(PyHeapTypeObject *type, PyObject *init, + unsigned int tp_version) +{ + if (!init || !tp_version) { + return 0; + } + int can_cache; + BEGIN_TYPE_LOCK(); + can_cache = ((PyTypeObject*)type)->tp_version_tag == tp_version; + #ifdef Py_GIL_DISABLED + can_cache = can_cache && _PyObject_HasDeferredRefcount(init); + #endif + if (can_cache) { + FT_ATOMIC_STORE_PTR_RELEASE(type->_spec_cache.init, init); + } + END_TYPE_LOCK(); + return can_cache; +} + static void set_flags(PyTypeObject *self, unsigned long mask, unsigned long flags) { @@ -11676,23 +11725,28 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, assert(_PyFrame_GetCode(cframe)->co_nlocalsplus > 0); PyObject *firstarg = PyStackRef_AsPyObjectBorrow(_PyFrame_GetLocalsArray(cframe)[0]); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; + } // The first argument might be a cell. - if (firstarg != NULL && (_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL)) { - // "firstarg" is a cell here unless (very unlikely) super() - // was called from the C-API before the first MAKE_CELL op. - if (_PyInterpreterFrame_LASTI(cframe) >= 0) { - // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need - // to use _PyOpcode_Deopt here: - assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || - _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); - assert(PyCell_Check(firstarg)); - firstarg = PyCell_GET(firstarg); + // "firstarg" is a cell here unless (very unlikely) super() + // was called from the C-API before the first MAKE_CELL op. + if ((_PyLocals_GetKind(co->co_localspluskinds, 0) & CO_FAST_CELL) && + (_PyInterpreterFrame_LASTI(cframe) >= 0)) { + // MAKE_CELL and COPY_FREE_VARS have no quickened forms, so no need + // to use _PyOpcode_Deopt here: + assert(_PyCode_CODE(co)[0].op.code == MAKE_CELL || + _PyCode_CODE(co)[0].op.code == COPY_FREE_VARS); + assert(PyCell_Check(firstarg)); + firstarg = PyCell_GetRef((PyCellObject *)firstarg); + if (firstarg == NULL) { + PyErr_SetString(PyExc_RuntimeError, "super(): arg[0] deleted"); + return -1; } } - if (firstarg == NULL) { - PyErr_SetString(PyExc_RuntimeError, - "super(): arg[0] deleted"); - return -1; + else { + Py_INCREF(firstarg); } // Look for __class__ in the free vars. @@ -11707,18 +11761,22 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, if (cell == NULL || !PyCell_Check(cell)) { PyErr_SetString(PyExc_RuntimeError, "super(): bad __class__ cell"); + Py_DECREF(firstarg); return -1; } - type = (PyTypeObject *) PyCell_GET(cell); + type = (PyTypeObject *) PyCell_GetRef((PyCellObject *)cell); if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): empty __class__ cell"); + Py_DECREF(firstarg); return -1; } if (!PyType_Check(type)) { PyErr_Format(PyExc_RuntimeError, "super(): __class__ is not a type (%s)", Py_TYPE(type)->tp_name); + Py_DECREF(type); + Py_DECREF(firstarg); return -1; } break; @@ -11727,6 +11785,7 @@ super_init_without_args(_PyInterpreterFrame *cframe, PyTypeObject **type_p, if (type == NULL) { PyErr_SetString(PyExc_RuntimeError, "super(): __class__ cell not found"); + Py_DECREF(firstarg); return -1; } @@ -11773,16 +11832,24 @@ super_init_impl(PyObject *self, PyTypeObject *type, PyObject *obj) { return -1; } } + else { + Py_INCREF(type); + Py_XINCREF(obj); + } - if (obj == Py_None) + if (obj == Py_None) { + Py_DECREF(obj); obj = NULL; + } if (obj != NULL) { obj_type = supercheck(type, obj); - if (obj_type == NULL) + if (obj_type == NULL) { + Py_DECREF(type); + Py_DECREF(obj); return -1; - Py_INCREF(obj); + } } - Py_XSETREF(su->type, (PyTypeObject*)Py_NewRef(type)); + Py_XSETREF(su->type, (PyTypeObject*)type); Py_XSETREF(su->obj, obj); Py_XSETREF(su->obj_type, obj_type); return 0; diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 562e3312b63e9a..33c4747bbef488 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -4979,39 +4979,229 @@ PyUnicode_DecodeUTF8(const char *s, #include "stringlib/codecs.h" #include "stringlib/undef.h" +#if (SIZEOF_SIZE_T == 8) /* Mask to quickly check whether a C 'size_t' contains a non-ASCII, UTF8-encoded char. */ -#if (SIZEOF_SIZE_T == 8) # define ASCII_CHAR_MASK 0x8080808080808080ULL +// used to count codepoints in UTF-8 string. +# define VECTOR_0101 0x0101010101010101ULL +# define VECTOR_00FF 0x00ff00ff00ff00ffULL #elif (SIZEOF_SIZE_T == 4) # define ASCII_CHAR_MASK 0x80808080U +# define VECTOR_0101 0x01010101U +# define VECTOR_00FF 0x00ff00ffU #else # error C 'size_t' size should be either 4 or 8! #endif +#if (defined(__clang__) || defined(__GNUC__)) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + return __builtin_ctzll((unsigned long long)v); +} +#elif defined(_MSC_VER) +#define HAVE_CTZ 1 +static inline unsigned int +ctz(size_t v) +{ + unsigned long pos; +#if SIZEOF_SIZE_T == 4 + _BitScanForward(&pos, v); +#else + _BitScanForward64(&pos, v); +#endif /* SIZEOF_SIZE_T */ + return pos; +} +#else +#define HAVE_CTZ 0 +#endif + +#if HAVE_CTZ && PY_LITTLE_ENDIAN +// load p[0]..p[size-1] as a size_t without unaligned access nor read ahead. +static size_t +load_unaligned(const unsigned char *p, size_t size) +{ + union { + size_t s; + unsigned char b[SIZEOF_SIZE_T]; + } u; + u.s = 0; + // This switch statement assumes little endian because: + // * union is faster than bitwise or and shift. + // * big endian machine is rare and hard to maintain. + switch (size) { + default: +#if SIZEOF_SIZE_T == 8 + case 8: + u.b[7] = p[7]; + _Py_FALLTHROUGH; + case 7: + u.b[6] = p[6]; + _Py_FALLTHROUGH; + case 6: + u.b[5] = p[5]; + _Py_FALLTHROUGH; + case 5: + u.b[4] = p[4]; + _Py_FALLTHROUGH; +#endif + case 4: + u.b[3] = p[3]; + _Py_FALLTHROUGH; + case 3: + u.b[2] = p[2]; + _Py_FALLTHROUGH; + case 2: + u.b[1] = p[1]; + _Py_FALLTHROUGH; + case 1: + u.b[0] = p[0]; + break; + case 0: + break; + } + return u.s; +} +#endif + +/* + * Find the first non-ASCII character in a byte sequence. + * + * This function scans a range of bytes from `start` to `end` and returns the + * index of the first byte that is not an ASCII character (i.e., has the most + * significant bit set). If all characters in the range are ASCII, it returns + * `end - start`. + */ static Py_ssize_t -ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +find_first_nonascii(const unsigned char *start, const unsigned char *end) +{ + const unsigned char *p = start; + + if (end - start >= SIZEOF_SIZE_T) { + const unsigned char *p2 = _Py_ALIGN_UP(p, SIZEOF_SIZE_T); +#if PY_LITTLE_ENDIAN && HAVE_CTZ + if (p < p2) { + size_t u; + memcpy(&u, p, sizeof(size_t)); + u &= ASCII_CHAR_MASK; + if (u) { + return (ctz(u) - 7) / 8; + } + p = p2; + } +#else /* PY_LITTLE_ENDIAN && HAVE_CTZ */ + while (p < p2) { + if (*p & 0x80) { + return p - start; + } + p++; + } +#endif + const unsigned char *e = end - SIZEOF_SIZE_T; + while (p <= e) { + size_t u = (*(const size_t *)p) & ASCII_CHAR_MASK; + if (u) { +#if PY_LITTLE_ENDIAN && HAVE_CTZ + return p - start + (ctz(u) - 7) / 8; +#else + // big endian and minor compilers are difficult to test. + // fallback to per byte check. + break; +#endif + } + p += SIZEOF_SIZE_T; + } + } +#if PY_LITTLE_ENDIAN && HAVE_CTZ + // we can not use *(const size_t*)p to avoid buffer overrun. + size_t u = load_unaligned(p, end - p) & ASCII_CHAR_MASK; + if (u) { + return p - start + (ctz(u) - 7) / 8; + } + return end - start; +#else + while (p < end) { + if (*p & 0x80) { + break; + } + p++; + } + return p - start; +#endif +} + +static inline int +scalar_utf8_start_char(unsigned int ch) { - const char *p = start; + // 0xxxxxxx or 11xxxxxx are first byte. + return (~ch >> 7 | ch >> 6) & 1; +} +static inline size_t +vector_utf8_start_chars(size_t v) +{ + return ((~v >> 7) | (v >> 6)) & VECTOR_0101; +} + + +// Count the number of UTF-8 code points in a given byte sequence. +static Py_ssize_t +utf8_count_codepoints(const unsigned char *s, const unsigned char *end) +{ + Py_ssize_t len = 0; + + if (end - s >= SIZEOF_SIZE_T) { + while (!_Py_IS_ALIGNED(s, ALIGNOF_SIZE_T)) { + len += scalar_utf8_start_char(*s++); + } + + while (s + SIZEOF_SIZE_T <= end) { + const unsigned char *e = end; + if (e - s > SIZEOF_SIZE_T * 255) { + e = s + SIZEOF_SIZE_T * 255; + } + Py_ssize_t vstart = 0; + while (s + SIZEOF_SIZE_T <= e) { + size_t v = *(size_t*)s; + size_t vs = vector_utf8_start_chars(v); + vstart += vs; + s += SIZEOF_SIZE_T; + } + vstart = (vstart & VECTOR_00FF) + ((vstart >> 8) & VECTOR_00FF); + vstart += vstart >> 16; +#if SIZEOF_SIZE_T == 8 + vstart += vstart >> 32; +#endif + len += vstart & 0x7ff; + } + } + while (s < end) { + len += scalar_utf8_start_char(*s++); + } + return len; +} + +static Py_ssize_t +ascii_decode(const char *start, const char *end, Py_UCS1 *dest) +{ #if SIZEOF_SIZE_T <= SIZEOF_VOID_P - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T) + if (_Py_IS_ALIGNED(start, ALIGNOF_SIZE_T) && _Py_IS_ALIGNED(dest, ALIGNOF_SIZE_T)) { /* Fast path, see in STRINGLIB(utf8_decode) for an explanation. */ - /* Help allocation */ - const char *_p = p; - Py_UCS1 * q = dest; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; + const char *p = start; + Py_UCS1 *q = dest; + while (p + SIZEOF_SIZE_T <= end) { + size_t value = *(const size_t *) p; if (value & ASCII_CHAR_MASK) break; *((size_t *)q) = value; - _p += SIZEOF_SIZE_T; + p += SIZEOF_SIZE_T; q += SIZEOF_SIZE_T; } - p = _p; while (p < end) { if ((unsigned char)*p & 0x80) break; @@ -5020,31 +5210,12 @@ ascii_decode(const char *start, const char *end, Py_UCS1 *dest) return p - start; } #endif - while (p < end) { - /* Fast path, see in STRINGLIB(utf8_decode) in stringlib/codecs.h - for an explanation. */ - if (_Py_IS_ALIGNED(p, ALIGNOF_SIZE_T)) { - /* Help allocation */ - const char *_p = p; - while (_p + SIZEOF_SIZE_T <= end) { - size_t value = *(const size_t *) _p; - if (value & ASCII_CHAR_MASK) - break; - _p += SIZEOF_SIZE_T; - } - p = _p; - if (_p == end) - break; - } - if ((unsigned char)*p & 0x80) - break; - ++p; - } - memcpy(dest, start, p - start); - return p - start; + Py_ssize_t pos = find_first_nonascii((const unsigned char*)start, + (const unsigned char*)end); + memcpy(dest, start, pos); + return pos; } - static int unicode_decode_utf8_impl(_PyUnicodeWriter *writer, const char *starts, const char *s, const char *end, @@ -5188,27 +5359,69 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, return get_latin1_char((unsigned char)s[0]); } - // fast path: try ASCII string. - const char *starts = s; - const char *end = s + size; - PyObject *u = PyUnicode_New(size, 127); - if (u == NULL) { + // I don't know this check is necessary or not. But there is a test + // case that requires size=PY_SSIZE_T_MAX cause MemoryError. + if (PY_SSIZE_T_MAX - sizeof(PyCompactUnicodeObject) < (size_t)size) { + PyErr_NoMemory(); return NULL; } - Py_ssize_t decoded = ascii_decode(s, end, PyUnicode_1BYTE_DATA(u)); - if (decoded == size) { + + const char *starts = s; + const char *end = s + size; + + Py_ssize_t pos = find_first_nonascii((const unsigned char*)starts, (const unsigned char*)end); + if (pos == size) { // fast path: ASCII string. + PyObject *u = PyUnicode_New(size, 127); + if (u == NULL) { + return NULL; + } + memcpy(PyUnicode_1BYTE_DATA(u), s, size); if (consumed) { *consumed = size; } return u; } - s += decoded; - size -= decoded; + + int maxchr = 127; + Py_ssize_t maxsize = size; + + unsigned char ch = (unsigned char)(s[pos]); + // error handler other than strict may remove/replace the invalid byte. + // consumed != NULL allows 1~3 bytes remainings. + // 0x80 <= ch < 0xc2 is invalid start byte that cause UnicodeDecodeError. + // otherwise: check the input and decide the maxchr and maxsize to reduce + // reallocation and copy. + if (error_handler == _Py_ERROR_STRICT && !consumed && ch >= 0xc2) { + // we only calculate the number of codepoints and don't determine the exact maxchr. + // This is because writing fast and portable SIMD code to find maxchr is difficult. + // If reallocation occurs for a larger maxchar, knowing the exact number of codepoints + // means that it is no longer necessary to allocate several times the required amount + // of memory. + maxsize = utf8_count_codepoints((const unsigned char *)s, (const unsigned char *)end); + if (ch < 0xc4) { // latin1 + maxchr = 0xff; + } + else if (ch < 0xf0) { // ucs2 + maxchr = 0xffff; + } + else { // ucs4 + maxchr = 0x10ffff; + } + } + PyObject *u = PyUnicode_New(maxsize, maxchr); + if (!u) { + return NULL; + } // Use _PyUnicodeWriter after fast path is failed. _PyUnicodeWriter writer; _PyUnicodeWriter_InitWithBuffer(&writer, u); - writer.pos = decoded; + if (maxchr <= 255) { + memcpy(PyUnicode_1BYTE_DATA(u), s, pos); + s += pos; + size -= pos; + writer.pos = pos; + } if (unicode_decode_utf8_impl(&writer, starts, s, end, error_handler, errors, @@ -5268,7 +5481,9 @@ PyUnicode_DecodeUTF8Stateful(const char *s, const char *errors, Py_ssize_t *consumed) { - return unicode_decode_utf8(s, size, _Py_ERROR_UNKNOWN, errors, consumed); + return unicode_decode_utf8(s, size, + errors ? _Py_ERROR_UNKNOWN : _Py_ERROR_STRICT, + errors, consumed); } @@ -15940,7 +16155,7 @@ encode_wstr_utf8(wchar_t *wstr, char **str, const char *name) int res; res = _Py_EncodeUTF8Ex(wstr, str, NULL, NULL, 1, _Py_ERROR_STRICT); if (res == -2) { - PyErr_Format(PyExc_RuntimeWarning, "cannot encode %s", name); + PyErr_Format(PyExc_RuntimeError, "cannot encode %s", name); return -1; } if (res < 0) { diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 17df9208f224f4..fb9868b3740b8c 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -13,6 +13,7 @@ #include "pycore_pythonrun.h" // _Py_SourceAsString() #include "pycore_sysmodule.h" // _PySys_GetAttr() #include "pycore_tuple.h" // _PyTuple_FromArray() +#include "pycore_cell.h" // PyCell_GetRef() #include "clinic/bltinmodule.c.h" @@ -209,7 +210,7 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *margs[3] = {name, bases, ns}; cls = PyObject_VectorcallDict(meta, margs, 3, mkw); if (cls != NULL && PyType_Check(cls) && PyCell_Check(cell)) { - PyObject *cell_cls = PyCell_GET(cell); + PyObject *cell_cls = PyCell_GetRef((PyCellObject *)cell); if (cell_cls != cls) { if (cell_cls == NULL) { const char *msg = @@ -221,9 +222,13 @@ builtin___build_class__(PyObject *self, PyObject *const *args, Py_ssize_t nargs, "__class__ set to %.200R defining %.200R as %.200R"; PyErr_Format(PyExc_TypeError, msg, cell_cls, name, cls); } + Py_XDECREF(cell_cls); Py_SETREF(cls, NULL); goto error; } + else { + Py_DECREF(cell_cls); + } } } error: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index a14b32b8108be8..d0e4c2bc45489b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -704,7 +704,7 @@ dummy_func( }; specializing op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -713,7 +713,7 @@ dummy_func( } OPCODE_DEFERRED_INC(BINARY_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_BINARY_SUBSCR, (container, sub -- res)) { @@ -790,11 +790,17 @@ dummy_func( // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; +#ifdef Py_GIL_DISABLED + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + DEOPT_IF(res_o == NULL); + STAT_INC(BINARY_SUBSCR, hit); +#else DEOPT_IF(index >= PyList_GET_SIZE(list)); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); +#endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); DEAD(sub_st); PyStackRef_CLOSE(list_st); @@ -1111,7 +1117,7 @@ dummy_func( }; specializing op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); @@ -1119,7 +1125,7 @@ dummy_func( } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_SEND, (receiver, v -- receiver, retval)) { @@ -1432,14 +1438,9 @@ dummy_func( inst(UNPACK_SEQUENCE_LIST, (unused/1, seq -- values[oparg])) { PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); DEOPT_IF(!PyList_CheckExact(seq_o)); - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - PyCriticalSection_Begin(&cs, seq_o); - #endif + DEOPT_IF(!LOCK_OBJECT(seq_o)); if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - PyCriticalSection_End(&cs); - #endif + UNLOCK_OBJECT(seq_o); DEOPT_IF(true); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -1447,9 +1448,7 @@ dummy_func( for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - PyCriticalSection_End(&cs); - #endif + UNLOCK_OBJECT(seq_o); DECREF_INPUTS(); } @@ -1947,7 +1946,7 @@ dummy_func( }; specializing op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super_st, class_st, unused -- global_super_st, class_st, unused)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -1956,7 +1955,7 @@ dummy_func( } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } tier1 op(_LOAD_SUPER_ATTR, (global_super_st, class_st, self_st -- attr, null if (oparg & 1))) { @@ -3330,7 +3329,7 @@ dummy_func( }; specializing op(_SPECIALIZE_CALL, (counter/1, callable[1], self_or_null[1], args[oparg] -- callable[1], self_or_null[1], args[oparg])) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_Call(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); @@ -3338,7 +3337,7 @@ dummy_func( } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } op(_MAYBE_EXPAND_METHOD, (callable[1], self_or_null[1], args[oparg] -- func[1], maybe_self[1], args[oparg])) { @@ -3723,10 +3722,10 @@ dummy_func( DEOPT_IF(!PyStackRef_IsNull(null[0])); DEOPT_IF(!PyType_Check(callable_o)); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != type_version); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)); STAT_INC(CALL, hit); @@ -3744,17 +3743,19 @@ dummy_func( _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); DEAD(init); DEAD(self); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); SYNC_SP(); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); ERROR_NO_POP(); } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -4001,8 +4002,10 @@ dummy_func( DEOPT_IF(callable_o != interp->callable_cache.list_append); assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o)); + DEOPT_IF(!LOCK_OBJECT(self_o)); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); ERROR_IF(err, error); @@ -4308,7 +4311,7 @@ dummy_func( assert(Py_TYPE(callable_o) == &PyFunction_Type); int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -4316,9 +4319,10 @@ dummy_func( // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. SYNC_SP(); - if (new_frame == NULL) { + if (temp == NULL) { ERROR_NO_POP(); } + new_frame = temp; } op(_CHECK_FUNCTION_VERSION_KW, (func_version/2, callable[1], self_or_null[1], unused[oparg], kwnames -- callable[1], self_or_null[1], unused[oparg], kwnames)) { @@ -4369,7 +4373,7 @@ dummy_func( _PUSH_FRAME; specializing op(_SPECIALIZE_CALL_KW, (counter/1, callable[1], self_or_null[1], args[oparg], kwnames -- callable[1], self_or_null[1], args[oparg], kwnames)) { - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _Py_Specialize_CallKw(callable[0], next_instr, oparg + !PyStackRef_IsNull(self_or_null[0])); @@ -4377,7 +4381,7 @@ dummy_func( } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } macro(CALL_KW) = diff --git a/Python/ceval.c b/Python/ceval.c index eba0f233a81ef3..5eda033eced628 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -28,7 +28,6 @@ #include "pycore_setobject.h" // _PySet_Update() #include "pycore_sliceobject.h" // _PyBuildSlice_ConsumeRefs #include "pycore_tuple.h" // _PyTuple_ITEMS() -#include "pycore_typeobject.h" // _PySuper_Lookup() #include "pycore_uop_ids.h" // Uops #include "pycore_pyerrors.h" @@ -818,6 +817,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int entry_frame.instr_ptr = (_Py_CODEUNIT *)_Py_INTERPRETER_TRAMPOLINE_INSTRUCTIONS + 1; entry_frame.stackpointer = entry_frame.localsplus; entry_frame.owner = FRAME_OWNED_BY_CSTACK; + entry_frame.visited = 0; entry_frame.return_offset = 0; /* Push frame */ entry_frame.previous = tstate->current_frame; @@ -2859,6 +2859,20 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } } + if (origin == NULL) { + // Fall back to __file__ for diagnostics if we don't have + // an origin that is a location + origin = PyModule_GetFilenameObject(v); + if (origin == NULL) { + if (!PyErr_ExceptionMatches(PyExc_SystemError)) { + goto done; + } + // PyModule_GetFilenameObject raised "module filename missing" + _PyErr_Clear(tstate); + } + assert(origin == NULL || PyUnicode_Check(origin)); + } + if (is_possibly_shadowing_stdlib) { assert(origin); errmsg = PyUnicode_FromFormat( @@ -2919,9 +2933,11 @@ _PyEval_ImportFrom(PyThreadState *tstate, PyObject *v, PyObject *name) } done_with_errmsg: - /* NULL checks for errmsg, mod_name, origin done by PyErr_SetImportError. */ - _PyErr_SetImportErrorWithNameFrom(errmsg, mod_name, origin, name); - Py_DECREF(errmsg); + if (errmsg != NULL) { + /* NULL checks for mod_name and origin done by _PyErr_SetImportErrorWithNameFrom */ + _PyErr_SetImportErrorWithNameFrom(errmsg, mod_name, origin, name); + Py_DECREF(errmsg); + } done: Py_XDECREF(origin); diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 7aaa045f375cf0..0a106ad636bfe8 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -342,6 +342,11 @@ _copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) return NULL; } + if (size != (Py_ssize_t)strlen(str)) { + PyErr_SetString(PyExc_ValueError, "found embedded NULL character"); + return NULL; + } + char *copied = PyMem_RawMalloc(size+1); if (copied == NULL) { PyErr_NoMemory(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index d46412a193332b..18f19773d25c90 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -981,6 +981,16 @@ JUMP_TO_JUMP_TARGET(); } Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (res_o == NULL) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } + STAT_INC(BINARY_SUBSCR, hit); + #else if (index >= PyList_GET_SIZE(list)) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -989,6 +999,7 @@ PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); + #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -1717,18 +1728,12 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_Begin(&cs, seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + if (!LOCK_OBJECT(seq_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); if (true) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); @@ -1739,11 +1744,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); @@ -4499,13 +4500,13 @@ JUMP_TO_JUMP_TARGET(); } PyTypeObject *tp = (PyTypeObject *)callable_o; - if (tp->tp_version_tag != type_version) { + if (FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version) { UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; if (!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize)) { UOP_STAT_INC(uopcode, miss); @@ -4536,25 +4537,29 @@ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; - stack_pointer += -1 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); JUMP_TO_ERROR(); } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, * as it will be checked after start_frame */ tstate->py_recursion_remaining--; + stack_pointer[0].bits = (uintptr_t)init_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } @@ -4907,8 +4912,13 @@ UOP_STAT_INC(uopcode, miss); JUMP_TO_JUMP_TARGET(); } + if (!LOCK_OBJECT(self_o)) { + UOP_STAT_INC(uopcode, miss); + JUMP_TO_JUMP_TARGET(); + } STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); if (err) JUMP_TO_ERROR(); @@ -5225,7 +5235,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -5233,12 +5243,15 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { JUMP_TO_ERROR(); } + new_frame = temp; + stack_pointer[0].bits = (uintptr_t)new_frame; + stack_pointer += 1; + assert(WITHIN_STACK_BOUNDS()); break; } diff --git a/Python/gc.c b/Python/gc.c index 63adecf0e05114..5b9588c8741b97 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -106,7 +106,7 @@ gc_old_space(PyGC_Head *g) } static inline int -flip_old_space(int space) +other_space(int space) { assert(space == 0 || space == 1); return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; @@ -430,24 +430,32 @@ validate_list(PyGC_Head *head, enum flagstates flags) #endif #ifdef GC_EXTRA_DEBUG + + static void -validate_old(GCState *gcstate) +gc_list_validate_space(PyGC_Head *head, int space) { + PyGC_Head *gc = GC_NEXT(head); + while (gc != head) { + assert(gc_old_space(gc) == space); + gc = GC_NEXT(gc); + } +} + +static void +validate_spaces(GCState *gcstate) { + int visited = gcstate->visited_space; + int not_visited = other_space(visited); + gc_list_validate_space(&gcstate->young.head, not_visited); for (int space = 0; space < 2; space++) { - PyGC_Head *head = &gcstate->old[space].head; - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - PyGC_Head *next = GC_NEXT(gc); - assert(gc_old_space(gc) == space); - gc = next; - } + gc_list_validate_space(&gcstate->old[space].head, space); } + gc_list_validate_space(&gcstate->permanent_generation.head, visited); } static void validate_consistent_old_space(PyGC_Head *head) { - PyGC_Head *prev = head; PyGC_Head *gc = GC_NEXT(head); if (gc == head) { return; @@ -457,23 +465,13 @@ validate_consistent_old_space(PyGC_Head *head) PyGC_Head *truenext = GC_NEXT(gc); assert(truenext != NULL); assert(gc_old_space(gc) == old_space); - prev = gc; gc = truenext; } - assert(prev == GC_PREV(head)); } -static void -gc_list_validate_space(PyGC_Head *head, int space) { - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - assert(gc_old_space(gc) == space); - gc = GC_NEXT(gc); - } -} #else -#define validate_old(g) do{}while(0) +#define validate_spaces(g) do{}while(0) #define validate_consistent_old_space(l) do{}while(0) #define gc_list_validate_space(l, s) do{}while(0) #endif @@ -494,7 +492,7 @@ update_refs(PyGC_Head *containers) next = GC_NEXT(gc); PyObject *op = FROM_GC(gc); if (_Py_IsImmortal(op)) { - gc_list_move(gc, &get_gc_state()->permanent_generation.head); + _PyObject_GC_UNTRACK(op); gc = next; continue; } @@ -733,13 +731,25 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) unreachable->_gc_next &= _PyGC_PREV_MASK; } +/* In theory, all tuples should be younger than the +* objects they refer to, as tuples are immortal. +* Therefore, untracking tuples in oldest-first order in the +* young generation before promoting them should have tracked +* all the tuples that can be untracked. +* +* Unfortunately, the C API allows tuples to be created +* and then filled in. So this won't untrack all tuples +* that can be untracked. It should untrack most of them +* and is much faster than a more complex approach that +* would untrack all relevant tuples. +*/ static void untrack_tuples(PyGC_Head *head) { - PyGC_Head *next, *gc = GC_NEXT(head); + PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyObject *op = FROM_GC(gc); - next = GC_NEXT(gc); + PyGC_Head *next = GC_NEXT(gc); if (PyTuple_CheckExact(op)) { _PyTuple_MaybeUntrack(op); } @@ -1293,8 +1303,10 @@ gc_collect_young(PyThreadState *tstate, struct gc_collection_stats *stats) { GCState *gcstate = &tstate->interp->gc; + validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(young); GC_STAT_ADD(0, collections, 1); #ifdef Py_STATS { @@ -1308,39 +1320,21 @@ gc_collect_young(PyThreadState *tstate, PyGC_Head survivors; gc_list_init(&survivors); + gc_list_set_space(young, gcstate->visited_space); gc_collect_region(tstate, young, &survivors, stats); - Py_ssize_t survivor_count = 0; - if (gcstate->visited_space) { - /* objects in visited space have bit set, so we set it here */ - survivor_count = gc_list_set_space(&survivors, 1); - } - else { - PyGC_Head *gc; - for (gc = GC_NEXT(&survivors); gc != &survivors; gc = GC_NEXT(gc)) { -#ifdef GC_DEBUG - assert(gc_old_space(gc) == 0); -#endif - survivor_count++; - } - } - (void)survivor_count; // Silence compiler warning gc_list_merge(&survivors, visited); - validate_old(gcstate); + validate_spaces(gcstate); gcstate->young.count = 0; gcstate->old[gcstate->visited_space].count++; - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; add_stats(gcstate, 0, stats); + validate_spaces(gcstate); } #ifndef NDEBUG static inline int IS_IN_VISITED(PyGC_Head *gc, int visited_space) { - assert(visited_space == 0 || flip_old_space(visited_space) == 0); + assert(visited_space == 0 || other_space(visited_space) == 0); return gc_old_space(gc) == visited_space; } #endif @@ -1348,7 +1342,7 @@ IS_IN_VISITED(PyGC_Head *gc, int visited_space) struct container_and_flag { PyGC_Head *container; int visited_space; - uintptr_t size; + intptr_t size; }; /* A traversal callback for adding to container) */ @@ -1371,7 +1365,7 @@ visit_add_to_container(PyObject *op, void *arg) return 0; } -static uintptr_t +static intptr_t expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) { struct container_and_flag arg = { @@ -1385,6 +1379,7 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat * have been marked as visited */ assert(IS_IN_VISITED(gc, gcstate->visited_space)); PyObject *op = FROM_GC(gc); + assert(_PyObject_GC_IS_TRACKED(op)); if (_Py_IsImmortal(op)) { PyGC_Head *next = GC_NEXT(gc); gc_list_move(gc, &get_gc_state()->permanent_generation.head); @@ -1402,22 +1397,191 @@ expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCStat /* Do bookkeeping for a completed GC cycle */ static void -completed_cycle(GCState *gcstate) -{ -#ifdef Py_DEBUG - PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; - assert(gc_list_is_empty(not_visited)); -#endif - gcstate->visited_space = flip_old_space(gcstate->visited_space); - /* Make sure all young objects have old space bit set correctly */ - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *gc = GC_NEXT(young); - while (gc != young) { - PyGC_Head *next = GC_NEXT(gc); - gc_set_old_space(gc, gcstate->visited_space); - gc = next; +completed_scavenge(GCState *gcstate) +{ + /* We must observe two invariants: + * 1. Members of the permanent generation must be marked visited. + * 2. We cannot touch members of the permanent generation. */ + int visited; + if (gc_list_is_empty(&gcstate->permanent_generation.head)) { + /* Permanent generation is empty so we can flip spaces bit */ + int not_visited = gcstate->visited_space; + visited = other_space(not_visited); + gcstate->visited_space = visited; + /* Make sure all objects have visited bit set correctly */ + gc_list_set_space(&gcstate->young.head, not_visited); } + else { + /* We must move the objects from visited to pending space. */ + visited = gcstate->visited_space; + int not_visited = other_space(visited); + assert(gc_list_is_empty(&gcstate->old[not_visited].head)); + gc_list_merge(&gcstate->old[visited].head, &gcstate->old[not_visited].head); + gc_list_set_space(&gcstate->old[not_visited].head, not_visited); + } + assert(gc_list_is_empty(&gcstate->old[visited].head)); gcstate->work_to_do = 0; + gcstate->phase = GC_PHASE_MARK; +} + +static intptr_t +move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) +{ + if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + gc_list_move(gc, reachable); + return 1; + } + } + return 0; +} + +static intptr_t +mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) +{ + // Transitively traverse all objects from reachable, until empty + struct container_and_flag arg = { + .container = reachable, + .visited_space = visited_space, + .size = 0 + }; + while (!gc_list_is_empty(reachable)) { + PyGC_Head *gc = _PyGCHead_NEXT(reachable); + assert(gc_old_space(gc) == visited_space); + gc_list_move(gc, visited); + PyObject *op = FROM_GC(gc); + traverseproc traverse = Py_TYPE(op)->tp_traverse; + (void) traverse(op, + visit_add_to_container, + &arg); + } + gc_list_validate_space(visited, visited_space); + return arg.size; +} + +static intptr_t +mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + // Move all objects on stacks to reachable + _PyRuntimeState *runtime = &_PyRuntime; + HEAD_LOCK(runtime); + PyThreadState* ts = PyInterpreterState_ThreadHead(interp); + HEAD_UNLOCK(runtime); + while (ts) { + _PyInterpreterFrame *frame = ts->current_frame; + while (frame) { + if (frame->owner == FRAME_OWNED_BY_CSTACK) { + frame = frame->previous; + continue; + } + _PyStackRef *locals = frame->localsplus; + _PyStackRef *sp = frame->stackpointer; + objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); + PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); + objects_marked += move_to_reachable(func, &reachable, visited_space); + while (sp > locals) { + sp--; + if (PyStackRef_IsNull(*sp)) { + continue; + } + PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); + if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { + PyGC_Head *gc = AS_GC(op); + if (_PyObject_GC_IS_TRACKED(op) && + gc_old_space(gc) != visited_space) { + gc_flip_old_space(gc); + objects_marked++; + gc_list_move(gc, &reachable); + } + } + } + if (!start && frame->visited) { + // If this frame has already been visited, then the lower frames + // will have already been visited and will not have changed + break; + } + frame->visited = 1; + frame = frame->previous; + } + HEAD_LOCK(runtime); + ts = PyThreadState_Next(ts); + HEAD_UNLOCK(runtime); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static intptr_t +mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) +{ + PyGC_Head reachable; + gc_list_init(&reachable); + Py_ssize_t objects_marked = 0; + objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); + objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); + objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); + struct types_state *types = &interp->types; + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); + } + for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); + objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); + } + objects_marked += mark_all_reachable(&reachable, visited, visited_space); + assert(gc_list_is_empty(&reachable)); + return objects_marked; +} + +static intptr_t +mark_at_start(PyThreadState *tstate) +{ + // TO DO -- Make this incremental + GCState *gcstate = &tstate->interp->gc; + PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); + objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); + gcstate->work_to_do -= objects_marked; + gcstate->phase = GC_PHASE_COLLECT; + validate_spaces(gcstate); + return objects_marked; +} + +static intptr_t +assess_work_to_do(GCState *gcstate) +{ + /* The amount of work we want to do depends on three things. + * 1. The number of new objects created + * 2. The growth in heap size since the last collection + * 3. The heap size (up to the number of new objects, to avoid quadratic effects) + * + * For a steady state heap, the amount of work to do is three times the number + * of new objects added to the heap. This ensures that we stay ahead in the + * worst case of all new objects being garbage. + * + * This could be improved by tracking survival rates, but it is still a + * large improvement on the non-marking approach. + */ + intptr_t scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + intptr_t new_objects = gcstate->young.count; + intptr_t max_heap_fraction = new_objects*3/2; + intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; + if (heap_fraction > max_heap_fraction) { + heap_fraction = max_heap_fraction; + } + gcstate->young.count = 0; + return new_objects + heap_fraction; } static void @@ -1425,18 +1589,30 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) { GC_STAT_ADD(1, collections, 1); GCState *gcstate = &tstate->interp->gc; + gcstate->work_to_do += assess_work_to_do(gcstate); + untrack_tuples(&gcstate->young.head); + if (gcstate->phase == GC_PHASE_MARK) { + Py_ssize_t objects_marked = mark_at_start(tstate); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + validate_spaces(gcstate); + return; + } PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; PyGC_Head increment; gc_list_init(&increment); - Py_ssize_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 1) { - scale_factor = 1; - } + int scale_factor = gcstate->old[0].threshold; + if (scale_factor < 2) { + scale_factor = 2; + } + intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); + GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); + gcstate->work_to_do -= objects_marked; + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &increment); - gcstate->young.count = 0; gc_list_validate_space(&increment, gcstate->visited_space); - Py_ssize_t increment_size = 0; + Py_ssize_t increment_size = gc_list_size(&increment); while (increment_size < gcstate->work_to_do) { if (gc_list_is_empty(not_visited)) { break; @@ -1444,54 +1620,56 @@ gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) PyGC_Head *gc = _PyGCHead_NEXT(not_visited); gc_list_move(gc, &increment); increment_size++; + assert(!_Py_IsImmortal(FROM_GC(gc))); gc_set_old_space(gc, gcstate->visited_space); increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); } + GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); validate_list(&increment, collecting_clear_unreachable_clear); gc_list_validate_space(&increment, gcstate->visited_space); PyGC_Head survivors; gc_list_init(&survivors); gc_collect_region(tstate, &increment, &survivors, stats); - gc_list_validate_space(&survivors, gcstate->visited_space); gc_list_merge(&survivors, visited); assert(gc_list_is_empty(&increment)); gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; gcstate->work_to_do -= increment_size; - validate_old(gcstate); add_stats(gcstate, 1, stats); if (gc_list_is_empty(not_visited)) { - completed_cycle(gcstate); + completed_scavenge(gcstate); } + validate_spaces(gcstate); } - static void gc_collect_full(PyThreadState *tstate, struct gc_collection_stats *stats) { GC_STAT_ADD(2, collections, 1); GCState *gcstate = &tstate->interp->gc; - validate_old(gcstate); + validate_spaces(gcstate); PyGC_Head *young = &gcstate->young.head; PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; + untrack_tuples(young); /* merge all generations into visited */ - gc_list_validate_space(young, gcstate->visited_space); - gc_list_set_space(pending, gcstate->visited_space); gc_list_merge(young, pending); + gc_list_validate_space(pending, 1-gcstate->visited_space); + gc_list_set_space(pending, gcstate->visited_space); gcstate->young.count = 0; gc_list_merge(pending, visited); + validate_spaces(gcstate); gc_collect_region(tstate, visited, visited, stats); + validate_spaces(gcstate); gcstate->young.count = 0; gcstate->old[0].count = 0; gcstate->old[1].count = 0; - - gcstate->work_to_do = - gcstate->young.threshold * 2; + completed_scavenge(gcstate); _PyGC_ClearAllFreeLists(tstate->interp); - validate_old(gcstate); + validate_spaces(gcstate); add_stats(gcstate, 2, stats); } @@ -1733,20 +1911,23 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - /* The permanent_generation has its old space bit set to zero */ - if (gcstate->visited_space) { - gc_list_set_space(&gcstate->young.head, 0); - } + /* The permanent_generation must be visited */ + gc_list_set_space(&gcstate->young.head, gcstate->visited_space); gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); gcstate->young.count = 0; PyGC_Head*old0 = &gcstate->old[0].head; PyGC_Head*old1 = &gcstate->old[1].head; + if (gcstate->visited_space) { + gc_list_set_space(old0, 1); + } + else { + gc_list_set_space(old1, 0); + } gc_list_merge(old0, &gcstate->permanent_generation.head); gcstate->old[0].count = 0; - gc_list_set_space(old1, 0); gc_list_merge(old1, &gcstate->permanent_generation.head); gcstate->old[1].count = 0; - validate_old(gcstate); + validate_spaces(gcstate); } void @@ -1754,8 +1935,8 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - &gcstate->old[0].head); - validate_old(gcstate); + &gcstate->old[gcstate->visited_space].head); + validate_spaces(gcstate); } Py_ssize_t @@ -1860,7 +2041,7 @@ _PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) _Py_stats->object_stats.object_visits = 0; } #endif - validate_old(gcstate); + validate_spaces(gcstate); _Py_atomic_store_int(&gcstate->collecting, 0); return stats.uncollectable + stats.collected; } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c9a5132269398c..fc0f55555f5c36 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -433,7 +433,7 @@ container = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT assert(frame->stackpointer == NULL); if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -444,7 +444,7 @@ } OPCODE_DEFERRED_INC(BINARY_SUBSCR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _BINARY_SUBSCR { @@ -577,11 +577,19 @@ // Deopt unless 0 <= sub < PyList_Size(list) DEOPT_IF(!_PyLong_IsNonNegativeCompact((PyLongObject *)sub), BINARY_SUBSCR); Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + #ifdef Py_GIL_DISABLED + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *res_o = _PyList_GetItemRef((PyListObject*)list, index); + stack_pointer = _PyFrame_GetStackPointer(frame); + DEOPT_IF(res_o == NULL, BINARY_SUBSCR); + STAT_INC(BINARY_SUBSCR, hit); + #else DEOPT_IF(index >= PyList_GET_SIZE(list), BINARY_SUBSCR); STAT_INC(BINARY_SUBSCR, hit); PyObject *res_o = PyList_GET_ITEM(list, index); assert(res_o != NULL); Py_INCREF(res_o); + #endif PyStackRef_CLOSE_SPECIALIZED(sub_st, (destructor)PyObject_Free); PyStackRef_CLOSE(list_st); res = PyStackRef_FromPyObjectSteal(res_o); @@ -872,7 +880,7 @@ callable = &stack_pointer[-2 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -882,7 +890,7 @@ } OPCODE_DEFERRED_INC(CALL); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD @@ -1040,10 +1048,10 @@ DEOPT_IF(!PyStackRef_IsNull(null[0]), CALL); DEOPT_IF(!PyType_Check(callable_o), CALL); PyTypeObject *tp = (PyTypeObject *)callable_o; - DEOPT_IF(tp->tp_version_tag != type_version, CALL); + DEOPT_IF(FT_ATOMIC_LOAD_UINT32_RELAXED(tp->tp_version_tag) != type_version, CALL); assert(tp->tp_flags & Py_TPFLAGS_INLINE_VALUES); PyHeapTypeObject *cls = (PyHeapTypeObject *)callable_o; - PyFunctionObject *init_func = (PyFunctionObject *)cls->_spec_cache.init; + PyFunctionObject *init_func = (PyFunctionObject *)FT_ATOMIC_LOAD_PTR_ACQUIRE(cls->_spec_cache.init); PyCodeObject *code = (PyCodeObject *)init_func->func_code; DEOPT_IF(!_PyThreadState_HasStackSpace(tstate, code->co_framesize + _Py_InitCleanup.co_framesize), CALL); STAT_INC(CALL, hit); @@ -1065,20 +1073,21 @@ _PyInterpreterFrame *shim = _PyFrame_PushTrampolineUnchecked( tstate, (PyCodeObject *)&_Py_InitCleanup, 1, frame); assert(_PyFrame_GetBytecode(shim)[0].op.code == EXIT_INIT_CHECK); + assert(_PyFrame_GetBytecode(shim)[1].op.code == RETURN_VALUE); stack_pointer = _PyFrame_GetStackPointer(frame); /* Push self onto stack of shim */ shim->localsplus[0] = PyStackRef_DUP(self[0]); _PyFrame_SetStackPointer(frame, stack_pointer); - init_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, init[0], NULL, args-1, oparg+1, NULL, shim); stack_pointer = _PyFrame_GetStackPointer(frame); - stack_pointer[-2 - oparg].bits = (uintptr_t)init_frame; - stack_pointer += -1 - oparg; + stack_pointer += -2 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (init_frame == NULL) { + if (temp == NULL) { _PyEval_FrameClearAndPop(tstate, shim); goto error; } + init_frame = temp; frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; /* Account for pushing the extra frame. * We don't check recursion depth here, @@ -1092,8 +1101,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -1888,7 +1895,7 @@ callable = &stack_pointer[-3 - oparg]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -1898,7 +1905,7 @@ } OPCODE_DEFERRED_INC(CALL_KW); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } /* Skip 2 cache entries */ // _MAYBE_EXPAND_METHOD_KW @@ -2086,7 +2093,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -2094,12 +2101,12 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2116,8 +2123,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2264,7 +2269,7 @@ int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable_o))->co_flags; PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable_o)); _PyFrame_SetStackPointer(frame, stack_pointer); - new_frame = _PyEvalFramePushAndInit( + _PyInterpreterFrame *temp = _PyEvalFramePushAndInit( tstate, callable[0], locals, args, positional_args, kwnames_o, frame ); @@ -2272,12 +2277,12 @@ PyStackRef_CLOSE(kwnames); // The frame has stolen all the arguments from the stack, // so there is no need to clean them up. - stack_pointer[-3 - oparg].bits = (uintptr_t)new_frame; - stack_pointer += -2 - oparg; + stack_pointer += -3 - oparg; assert(WITHIN_STACK_BOUNDS()); - if (new_frame == NULL) { + if (temp == NULL) { goto error; } + new_frame = temp; } // _SAVE_RETURN_OFFSET { @@ -2294,8 +2299,6 @@ // Eventually this should be the only occurrence of this code. assert(tstate->interp->eval_frame == NULL); _PyInterpreterFrame *temp = new_frame; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); assert(new_frame->previous == frame || new_frame->previous->previous == frame); CALL_STAT_INC(inlined_py_calls); @@ -2375,8 +2378,10 @@ DEOPT_IF(callable_o != interp->callable_cache.list_append, CALL); assert(self_o != NULL); DEOPT_IF(!PyList_Check(self_o), CALL); + DEOPT_IF(!LOCK_OBJECT(self_o), CALL); STAT_INC(CALL, hit); int err = _PyList_AppendTakeRef((PyListObject *)self_o, PyStackRef_AsPyObjectSteal(arg)); + UNLOCK_OBJECT(self_o); PyStackRef_CLOSE(self); PyStackRef_CLOSE(callable); if (err) goto pop_3_error; @@ -6322,7 +6327,7 @@ global_super_st = stack_pointer[-3]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT int load_method = oparg & 1; if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; @@ -6333,7 +6338,7 @@ } OPCODE_DEFERRED_INC(LOAD_SUPER_ATTR); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _LOAD_SUPER_ATTR { @@ -7057,7 +7062,7 @@ receiver = stack_pointer[-2]; uint16_t counter = read_u16(&this_instr[1].cache); (void)counter; - #if ENABLE_SPECIALIZATION + #if ENABLE_SPECIALIZATION_FT if (ADAPTIVE_COUNTER_TRIGGERS(counter)) { next_instr = this_instr; _PyFrame_SetStackPointer(frame, stack_pointer); @@ -7067,7 +7072,7 @@ } OPCODE_DEFERRED_INC(SEND); ADVANCE_ADAPTIVE_COUNTER(this_instr[1].counter); - #endif /* ENABLE_SPECIALIZATION */ + #endif /* ENABLE_SPECIALIZATION_FT */ } // _SEND { @@ -8032,18 +8037,9 @@ values = &stack_pointer[-1]; PyObject *seq_o = PyStackRef_AsPyObjectBorrow(seq); DEOPT_IF(!PyList_CheckExact(seq_o), UNPACK_SEQUENCE); - #ifdef Py_GIL_DISABLED - PyCriticalSection cs; - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_Begin(&cs, seq_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + DEOPT_IF(!LOCK_OBJECT(seq_o), UNPACK_SEQUENCE); if (PyList_GET_SIZE(seq_o) != oparg) { - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); DEOPT_IF(true, UNPACK_SEQUENCE); } STAT_INC(UNPACK_SEQUENCE, hit); @@ -8051,11 +8047,7 @@ for (int i = oparg; --i >= 0; ) { *values++ = PyStackRef_FromPyObjectNew(items[i]); } - #ifdef Py_GIL_DISABLED - _PyFrame_SetStackPointer(frame, stack_pointer); - PyCriticalSection_End(&cs); - stack_pointer = _PyFrame_GetStackPointer(frame); - #endif + UNLOCK_OBJECT(seq_o); PyStackRef_CLOSE(seq); stack_pointer += -1 + oparg; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/import.c b/Python/import.c index 09fe95fa1fb647..f3511aaf7b8010 100644 --- a/Python/import.c +++ b/Python/import.c @@ -749,7 +749,7 @@ const char * _PyImport_ResolveNameWithPackageContext(const char *name) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif if (PKGCONTEXT != NULL) { const char *p = strrchr(PKGCONTEXT, '.'); @@ -759,7 +759,7 @@ _PyImport_ResolveNameWithPackageContext(const char *name) } } #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return name; } @@ -768,12 +768,12 @@ const char * _PyImport_SwapPackageContext(const char *newcontext) { #ifndef HAVE_THREAD_LOCAL - PyThread_acquire_lock(EXTENSIONS.mutex, WAIT_LOCK); + PyMutex_Lock(&EXTENSIONS.mutex); #endif const char *oldcontext = PKGCONTEXT; PKGCONTEXT = newcontext; #ifndef HAVE_THREAD_LOCAL - PyThread_release_lock(EXTENSIONS.mutex); + PyMutex_Unlock(&EXTENSIONS.mutex); #endif return oldcontext; } @@ -1157,12 +1157,14 @@ del_extensions_cache_value(struct extensions_cache_value *value) static void * hashtable_key_from_2_strings(PyObject *str1, PyObject *str2, const char sep) { - Py_ssize_t str1_len, str2_len; - const char *str1_data = PyUnicode_AsUTF8AndSize(str1, &str1_len); - const char *str2_data = PyUnicode_AsUTF8AndSize(str2, &str2_len); + const char *str1_data = _PyUnicode_AsUTF8NoNUL(str1); + const char *str2_data = _PyUnicode_AsUTF8NoNUL(str2); if (str1_data == NULL || str2_data == NULL) { return NULL; } + Py_ssize_t str1_len = strlen(str1_data); + Py_ssize_t str2_len = strlen(str2_data); + /* Make sure sep and the NULL byte won't cause an overflow. */ assert(SIZE_MAX - str1_len - str2_len > 2); size_t size = str1_len + 1 + str2_len + 1; diff --git a/Python/initconfig.c b/Python/initconfig.c index 438f8a5c1cf1ce..7851b86db1f6d0 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -168,6 +168,9 @@ static const PyConfigSpec PYCONFIG_SPEC[] = { SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), +#ifdef __APPLE__ + SPEC(use_system_logger, BOOL, PUBLIC, NO_SYS), +#endif SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), @@ -884,6 +887,9 @@ config_check_consistency(const PyConfig *config) assert(config->cpu_count != 0); // config->use_frozen_modules is initialized later // by _PyConfig_InitImportConfig(). +#ifdef __APPLE__ + assert(config->use_system_logger >= 0); +#endif #ifdef Py_STATS assert(config->_pystats >= 0); #endif @@ -986,6 +992,9 @@ _PyConfig_InitCompatConfig(PyConfig *config) config->_is_python_build = 0; config->code_debug_ranges = 1; config->cpu_count = -1; +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif #ifdef Py_GIL_DISABLED config->enable_gil = _PyConfig_GIL_DEFAULT; config->tlbc_enabled = 1; @@ -1015,6 +1024,9 @@ config_init_defaults(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } @@ -1049,6 +1061,9 @@ PyConfig_InitIsolatedConfig(PyConfig *config) #ifdef MS_WINDOWS config->legacy_windows_stdio = 0; #endif +#ifdef __APPLE__ + config->use_system_logger = 0; +#endif } diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index f144f7d436fe68..ad077dc861b0a7 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -484,11 +484,11 @@ _PyPerfTrampoline_Init(int activate) return -1; } if (!activate) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); perf_status = PERF_STATUS_NO_INIT; } else { - tstate->interp->eval_frame = py_trampoline_evaluator; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); if (new_code_arena() < 0) { return -1; } @@ -514,7 +514,7 @@ _PyPerfTrampoline_Fini(void) } PyThreadState *tstate = _PyThreadState_GET(); if (tstate->interp->eval_frame == py_trampoline_evaluator) { - tstate->interp->eval_frame = NULL; + _PyInterpreterState_SetEvalFrameFunc(tstate->interp, NULL); } if (perf_status == PERF_STATUS_OK) { trampoline_api.free_state(trampoline_api.state); diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 23882d083844ac..06418123d6dd9b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -45,7 +45,9 @@ #endif #if defined(__APPLE__) +# include # include +# include #endif #ifdef HAVE_SIGNAL_H @@ -75,6 +77,9 @@ static PyStatus init_sys_streams(PyThreadState *tstate); #ifdef __ANDROID__ static PyStatus init_android_streams(PyThreadState *tstate); #endif +#if defined(__APPLE__) +static PyStatus init_apple_streams(PyThreadState *tstate); +#endif static void wait_for_thread_shutdown(PyThreadState *tstate); static void finalize_subinterpreters(void); static void call_ll_exitfuncs(_PyRuntimeState *runtime); @@ -1257,6 +1262,14 @@ init_interp_main(PyThreadState *tstate) return status; } #endif +#if defined(__APPLE__) + if (config->use_system_logger) { + status = init_apple_streams(tstate); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + } +#endif #ifdef Py_DEBUG run_presite(tstate); @@ -1888,7 +1901,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyXI_Fini(tstate->interp); _PyExc_ClearExceptionGroupType(tstate->interp); _Py_clear_generic_types(tstate->interp); - _PyDtoa_Fini(tstate->interp); /* Clear interpreter state and all thread states */ _PyInterpreterState_Clear(tstate); @@ -1910,6 +1922,9 @@ finalize_interp_clear(PyThreadState *tstate) finalize_interp_types(tstate->interp); + /* Finalize dtoa at last so that finalizers calling repr of float doesn't crash */ + _PyDtoa_Fini(tstate->interp); + /* Free any delayed free requests immediately */ _PyMem_FiniDelayed(tstate->interp); @@ -2931,6 +2946,75 @@ init_android_streams(PyThreadState *tstate) #endif // __ANDROID__ +#if defined(__APPLE__) + +static PyObject * +apple_log_write_impl(PyObject *self, PyObject *args) +{ + int logtype = 0; + const char *text = NULL; + if (!PyArg_ParseTuple(args, "iy", &logtype, &text)) { + return NULL; + } + + // Call the underlying Apple logging API. The os_log unified logging APIs + // were introduced in macOS 10.12, iOS 10.0, tvOS 10.0, and watchOS 3.0; + // this call is a no-op on older versions. + #if TARGET_OS_IPHONE || (TARGET_OS_OSX && MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_12) + // Pass the user-provided text through explicit %s formatting + // to avoid % literals being interpreted as a formatting directive. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + #endif + Py_RETURN_NONE; +} + + +static PyMethodDef apple_log_write_method = { + "apple_log_write", apple_log_write_impl, METH_VARARGS +}; + + +static PyStatus +init_apple_streams(PyThreadState *tstate) +{ + PyStatus status = _PyStatus_OK(); + PyObject *_apple_support = NULL; + PyObject *apple_log_write = NULL; + PyObject *result = NULL; + + _apple_support = PyImport_ImportModule("_apple_support"); + if (_apple_support == NULL) { + goto error; + } + + apple_log_write = PyCFunction_New(&apple_log_write_method, NULL); + if (apple_log_write == NULL) { + goto error; + } + + // Initialize the logging streams, sending stdout -> Default; stderr -> Error + result = PyObject_CallMethod( + _apple_support, "init_streams", "Oii", + apple_log_write, OS_LOG_TYPE_DEFAULT, OS_LOG_TYPE_ERROR); + if (result == NULL) { + goto error; + } + + goto done; + +error: + _PyErr_Print(tstate); + status = _PyStatus_ERR("failed to initialize Apple log streams"); + +done: + Py_XDECREF(result); + Py_XDECREF(apple_log_write); + Py_XDECREF(_apple_support); + return status; +} + +#endif // __APPLE__ + static void _Py_FatalError_DumpTracebacks(int fd, PyInterpreterState *interp, diff --git a/Python/pystate.c b/Python/pystate.c index 3ceae229f75cd0..839413a65a42fb 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -2838,7 +2838,9 @@ _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, } #endif RARE_EVENT_INC(set_eval_frame_func); + _PyEval_StopTheWorld(interp); interp->eval_frame = eval_frame; + _PyEval_StartTheWorld(interp); } diff --git a/Python/specialize.c b/Python/specialize.c index 172dae7d374602..fd182e7d7a9215 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -231,6 +231,8 @@ print_gc_stats(FILE *out, GCStats *stats) fprintf(out, "GC[%d] collections: %" PRIu64 "\n", i, stats[i].collections); fprintf(out, "GC[%d] object visits: %" PRIu64 "\n", i, stats[i].object_visits); fprintf(out, "GC[%d] objects collected: %" PRIu64 "\n", i, stats[i].objects_collected); + fprintf(out, "GC[%d] objects reachable from roots: %" PRIu64 "\n", i, stats[i].objects_transitively_reachable); + fprintf(out, "GC[%d] objects not reachable from roots: %" PRIu64 "\n", i, stats[i].objects_not_transitively_reachable); } } @@ -792,9 +794,8 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P PyObject *global_super = PyStackRef_AsPyObjectBorrow(global_super_st); PyObject *cls = PyStackRef_AsPyObjectBorrow(cls_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[LOAD_SUPER_ATTR] == INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR); - _PySuperAttrCache *cache = (_PySuperAttrCache *)(instr + 1); if (global_super != (PyObject *)&PySuper_Type) { SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_SHADOWED); goto fail; @@ -803,19 +804,11 @@ _Py_Specialize_LoadSuperAttr(_PyStackRef global_super_st, _PyStackRef cls_st, _P SPECIALIZATION_FAIL(LOAD_SUPER_ATTR, SPEC_FAIL_SUPER_BAD_CLASS); goto fail; } - instr->op.code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; - goto success; - -fail: - STAT_INC(LOAD_SUPER_ATTR, failure); - assert(!PyErr_Occurred()); - instr->op.code = LOAD_SUPER_ATTR; - cache->counter = adaptive_counter_backoff(cache->counter); + uint8_t load_code = load_method ? LOAD_SUPER_ATTR_METHOD : LOAD_SUPER_ATTR_ATTR; + specialize(instr, load_code); return; -success: - STAT_INC(LOAD_SUPER_ATTR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); +fail: + unspecialize(instr); } typedef enum { @@ -954,7 +947,10 @@ specialize_dict_access( return 0; } _PyAttrCache *cache = (_PyAttrCache *)(instr + 1); - if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && _PyObject_InlineValues(owner)->valid) { + if (type->tp_flags & Py_TPFLAGS_INLINE_VALUES && + _PyObject_InlineValues(owner)->valid && + !(base_op == STORE_ATTR && _PyObject_GetManagedDict(owner) != NULL)) + { PyDictKeysObject *keys = ((PyHeapTypeObject *)type)->ht_cached_keys; assert(PyUnicode_CheckExact(name)); Py_ssize_t index = _PyDictKeys_StringLookup(keys, name); @@ -1717,15 +1713,15 @@ _Py_Specialize_BinarySubscr( PyObject *container = PyStackRef_AsPyObjectBorrow(container_st); PyObject *sub = PyStackRef_AsPyObjectBorrow(sub_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[BINARY_SUBSCR] == INLINE_CACHE_ENTRIES_BINARY_SUBSCR); - _PyBinarySubscrCache *cache = (_PyBinarySubscrCache *)(instr + 1); PyTypeObject *container_type = Py_TYPE(container); + uint8_t specialized_op; if (container_type == &PyList_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_LIST_INT; + specialized_op = BINARY_SUBSCR_LIST_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1738,7 +1734,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyTuple_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_TUPLE_INT; + specialized_op = BINARY_SUBSCR_TUPLE_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1751,7 +1747,7 @@ _Py_Specialize_BinarySubscr( if (container_type == &PyUnicode_Type) { if (PyLong_CheckExact(sub)) { if (_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) { - instr->op.code = BINARY_SUBSCR_STR_INT; + specialized_op = BINARY_SUBSCR_STR_INT; goto success; } SPECIALIZATION_FAIL(BINARY_SUBSCR, SPEC_FAIL_OUT_OF_RANGE); @@ -1762,9 +1758,10 @@ _Py_Specialize_BinarySubscr( goto fail; } if (container_type == &PyDict_Type) { - instr->op.code = BINARY_SUBSCR_DICT; + specialized_op = BINARY_SUBSCR_DICT; goto success; } +#ifndef Py_GIL_DISABLED PyTypeObject *cls = Py_TYPE(container); PyObject *descriptor = _PyType_Lookup(cls, &_Py_ID(__getitem__)); if (descriptor && Py_TYPE(descriptor) == &PyFunction_Type) { @@ -1797,28 +1794,25 @@ _Py_Specialize_BinarySubscr( // struct _specialization_cache): ht->_spec_cache.getitem = descriptor; ht->_spec_cache.getitem_version = version; - instr->op.code = BINARY_SUBSCR_GETITEM; + specialized_op = BINARY_SUBSCR_GETITEM; goto success; } +#endif // Py_GIL_DISABLED SPECIALIZATION_FAIL(BINARY_SUBSCR, binary_subscr_fail_kind(container_type, sub)); fail: - STAT_INC(BINARY_SUBSCR, failure); - assert(!PyErr_Occurred()); - instr->op.code = BINARY_SUBSCR; - cache->counter = adaptive_counter_backoff(cache->counter); + unspecialize(instr); return; success: - STAT_INC(BINARY_SUBSCR, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + specialize(instr, specialized_op); } #ifdef Py_STATS static int -store_subscr_fail_kind(PyObject *container_type) +store_subscr_fail_kind(PyObject *container, PyObject *sub) { + PyTypeObject *container_type = Py_TYPE(container); PyMappingMethods *as_mapping = container_type->tp_as_mapping; if (as_mapping && (as_mapping->mp_ass_subscript == PyDict_Type.tp_as_mapping->mp_ass_subscript)) { @@ -1916,42 +1910,42 @@ _Py_Specialize_StoreSubscr(_PyStackRef container_st, _PyStackRef sub_st, _Py_COD specialize(instr, STORE_SUBSCR_DICT); return; } - SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container_type)); + SPECIALIZATION_FAIL(STORE_SUBSCR, store_subscr_fail_kind(container, sub)); unspecialize(instr); } -/* Returns a borrowed reference. - * The reference is only valid if guarded by a type version check. - */ -static PyFunctionObject * -get_init_for_simple_managed_python_class(PyTypeObject *tp) +/* Returns a strong reference. */ +static PyObject * +get_init_for_simple_managed_python_class(PyTypeObject *tp, unsigned int *tp_version) { assert(tp->tp_new == PyBaseObject_Type.tp_new); if (tp->tp_alloc != PyType_GenericAlloc) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OVERRIDDEN); return NULL; } - if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { + unsigned long tp_flags = PyType_GetFlags(tp); + if ((tp_flags & Py_TPFLAGS_INLINE_VALUES) == 0) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_INLINE_VALUES); return NULL; } - if (!(tp->tp_flags & Py_TPFLAGS_HEAPTYPE)) { + if (!(tp_flags & Py_TPFLAGS_HEAPTYPE)) { /* Is this possible? */ SPECIALIZATION_FAIL(CALL, SPEC_FAIL_EXPECTED_ERROR); return NULL; } - PyObject *init = _PyType_Lookup(tp, &_Py_ID(__init__)); + PyObject *init = _PyType_LookupRefAndVersion(tp, &_Py_ID(__init__), tp_version); if (init == NULL || !PyFunction_Check(init)) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_PYTHON); + Py_XDECREF(init); return NULL; } int kind = function_kind((PyCodeObject *)PyFunction_GET_CODE(init)); if (kind != SIMPLE_FUNCTION) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_INIT_NOT_SIMPLE); + Py_DECREF(init); return NULL; } - ((PyHeapTypeObject *)tp)->_spec_cache.init = init; - return (PyFunctionObject *)init; + return init; } static int @@ -1963,20 +1957,20 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) int oparg = instr->op.arg; if (nargs == 1 && oparg == 1) { if (tp == &PyUnicode_Type) { - instr->op.code = CALL_STR_1; + specialize(instr, CALL_STR_1); return 0; } else if (tp == &PyType_Type) { - instr->op.code = CALL_TYPE_1; + specialize(instr, CALL_TYPE_1); return 0; } else if (tp == &PyTuple_Type) { - instr->op.code = CALL_TUPLE_1; + specialize(instr, CALL_TUPLE_1); return 0; } } if (tp->tp_vectorcall != NULL) { - instr->op.code = CALL_BUILTIN_CLASS; + specialize(instr, CALL_BUILTIN_CLASS); return 0; } goto generic; @@ -1985,19 +1979,25 @@ specialize_class_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) goto generic; } if (tp->tp_new == PyBaseObject_Type.tp_new) { - PyFunctionObject *init = get_init_for_simple_managed_python_class(tp); - if (type_get_version(tp, CALL) == 0) { + unsigned int tp_version = 0; + PyObject *init = get_init_for_simple_managed_python_class(tp, &tp_version); + if (!tp_version) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_OUT_OF_VERSIONS); + Py_XDECREF(init); return -1; } - if (init != NULL) { + if (init != NULL && _PyType_CacheInitForSpecialization( + (PyHeapTypeObject *)tp, init, tp_version)) { _PyCallCache *cache = (_PyCallCache *)(instr + 1); - write_u32(cache->func_version, tp->tp_version_tag); - _Py_SET_OPCODE(*instr, CALL_ALLOC_AND_ENTER_INIT); + write_u32(cache->func_version, tp_version); + specialize(instr, CALL_ALLOC_AND_ENTER_INIT); + Py_DECREF(init); return 0; } + Py_XDECREF(init); } generic: - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } @@ -2013,7 +2013,7 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_WRONG_NUMBER_ARGUMENTS); return -1; } - instr->op.code = CALL_METHOD_DESCRIPTOR_NOARGS; + specialize(instr, CALL_METHOD_DESCRIPTOR_NOARGS); return 0; } case METH_O: { @@ -2027,22 +2027,22 @@ specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr, bool pop = (next.op.code == POP_TOP); int oparg = instr->op.arg; if ((PyObject *)descr == list_append && oparg == 1 && pop) { - instr->op.code = CALL_LIST_APPEND; + specialize(instr, CALL_LIST_APPEND); return 0; } - instr->op.code = CALL_METHOD_DESCRIPTOR_O; + specialize(instr, CALL_METHOD_DESCRIPTOR_O); return 0; } case METH_FASTCALL: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS; + specialize(instr, CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS); return 0; } } - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } @@ -2072,12 +2072,15 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); + uint8_t opcode; if (argcount == nargs + bound_method) { - instr->op.code = bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; + opcode = + bound_method ? CALL_BOUND_METHOD_EXACT_ARGS : CALL_PY_EXACT_ARGS; } else { - instr->op.code = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; + opcode = bound_method ? CALL_BOUND_METHOD_GENERAL : CALL_PY_GENERAL; } + specialize(instr, opcode); return 0; } @@ -2104,7 +2107,7 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, return -1; } write_u32(cache->func_version, version); - instr->op.code = bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY; + specialize(instr, bound_method ? CALL_KW_BOUND_METHOD : CALL_KW_PY); return 0; } @@ -2126,10 +2129,10 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* len(o) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.len) { - instr->op.code = CALL_LEN; + specialize(instr, CALL_LEN); return 0; } - instr->op.code = CALL_BUILTIN_O; + specialize(instr, CALL_BUILTIN_O); return 0; } case METH_FASTCALL: { @@ -2137,19 +2140,19 @@ specialize_c_call(PyObject *callable, _Py_CODEUNIT *instr, int nargs) /* isinstance(o1, o2) */ PyInterpreterState *interp = _PyInterpreterState_GET(); if (callable == interp->callable_cache.isinstance) { - instr->op.code = CALL_ISINSTANCE; + specialize(instr, CALL_ISINSTANCE); return 0; } } - instr->op.code = CALL_BUILTIN_FAST; + specialize(instr, CALL_BUILTIN_FAST); return 0; } case METH_FASTCALL | METH_KEYWORDS: { - instr->op.code = CALL_BUILTIN_FAST_WITH_KEYWORDS; + specialize(instr, CALL_BUILTIN_FAST_WITH_KEYWORDS); return 0; } default: - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); return 0; } } @@ -2159,10 +2162,9 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CALL] == INLINE_CACHE_ENTRIES_CALL); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyCFunction_CheckExact(callable)) { fail = specialize_c_call(callable, instr, nargs); @@ -2187,19 +2189,11 @@ _Py_Specialize_Call(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } else { - instr->op.code = CALL_NON_PY_GENERAL; + specialize(instr, CALL_NON_PY_GENERAL); fail = 0; } if (fail) { - STAT_INC(CALL, failure); - assert(!PyErr_Occurred()); - instr->op.code = CALL; - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } } @@ -2208,10 +2202,9 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) { PyObject *callable = PyStackRef_AsPyObjectBorrow(callable_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[CALL_KW] == INLINE_CACHE_ENTRIES_CALL_KW); assert(_Py_OPCODE(*instr) != INSTRUMENTED_CALL_KW); - _PyCallCache *cache = (_PyCallCache *)(instr + 1); int fail; if (PyFunction_Check(callable)) { fail = specialize_py_call_kw((PyFunctionObject *)callable, instr, nargs, false); @@ -2227,19 +2220,11 @@ _Py_Specialize_CallKw(_PyStackRef callable_st, _Py_CODEUNIT *instr, int nargs) } } else { - instr->op.code = CALL_KW_NON_PY; + specialize(instr, CALL_KW_NON_PY); fail = 0; } if (fail) { - STAT_INC(CALL, failure); - assert(!PyErr_Occurred()); - instr->op.code = CALL_KW; - cache->counter = adaptive_counter_backoff(cache->counter); - } - else { - STAT_INC(CALL, success); - assert(!PyErr_Occurred()); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } } @@ -2636,28 +2621,21 @@ _Py_Specialize_Send(_PyStackRef receiver_st, _Py_CODEUNIT *instr) { PyObject *receiver = PyStackRef_AsPyObjectBorrow(receiver_st); - assert(ENABLE_SPECIALIZATION); + assert(ENABLE_SPECIALIZATION_FT); assert(_PyOpcode_Caches[SEND] == INLINE_CACHE_ENTRIES_SEND); - _PySendCache *cache = (_PySendCache *)(instr + 1); PyTypeObject *tp = Py_TYPE(receiver); if (tp == &PyGen_Type || tp == &PyCoro_Type) { if (_PyInterpreterState_GET()->eval_frame) { SPECIALIZATION_FAIL(SEND, SPEC_FAIL_OTHER); goto failure; } - instr->op.code = SEND_GEN; - goto success; + specialize(instr, SEND_GEN); + return; } SPECIALIZATION_FAIL(SEND, _PySpecialization_ClassifyIterator(receiver)); failure: - STAT_INC(SEND, failure); - instr->op.code = SEND; - cache->counter = adaptive_counter_backoff(cache->counter); - return; -success: - STAT_INC(SEND, success); - cache->counter = adaptive_counter_cooldown(); + unspecialize(instr); } #ifdef Py_STATS @@ -2809,6 +2787,16 @@ _Py_Specialize_ContainsOp(_PyStackRef value_st, _Py_CODEUNIT *instr) * Ends with a RESUME so that it is not traced. * This is used as a plain code object, not a function, * so must not access globals or builtins. + * There are a few other constraints imposed on the code + * by the free-threaded build: + * + * 1. The RESUME instruction must not be executed. Otherwise we may attempt to + * free the statically allocated TLBC array. + * 2. It must contain no specializable instructions. Specializing multiple + * copies of the same bytecode is not thread-safe in free-threaded builds. + * + * This should be dynamically allocated if either of those restrictions need to + * be lifted. */ #define NO_LOC_4 (128 | (PY_CODE_LOCATION_INFO_NONE << 3) | 3) @@ -2818,6 +2806,13 @@ static const PyBytesObject no_location = { .ob_sval = { NO_LOC_4 } }; +#ifdef Py_GIL_DISABLED +static _PyCodeArray init_cleanup_tlbc = { + .size = 1, + .entries = {(char*) &_Py_InitCleanup.co_code_adaptive}, +}; +#endif + const struct _PyCode8 _Py_InitCleanup = { _PyVarObject_HEAD_INIT(&PyCode_Type, 3), .co_consts = (PyObject *)&_Py_SINGLETON(tuple_empty), @@ -2833,6 +2828,9 @@ const struct _PyCode8 _Py_InitCleanup = { ._co_firsttraceable = 4, .co_stacksize = 2, .co_framesize = 2 + FRAME_SPECIALS_SIZE, +#ifdef Py_GIL_DISABLED + .co_tlbc = &init_cleanup_tlbc, +#endif .co_code_adaptive = { EXIT_INIT_CHECK, 0, RETURN_VALUE, 0, diff --git a/Python/stdlib_module_names.h b/Python/stdlib_module_names.h index c8cdb933bb108f..584b050fc4bb6e 100644 --- a/Python/stdlib_module_names.h +++ b/Python/stdlib_module_names.h @@ -6,6 +6,7 @@ static const char* _Py_stdlib_module_names[] = { "_abc", "_aix_support", "_android_support", +"_apple_support", "_ast", "_asyncio", "_bisect", diff --git a/Python/structmember.c b/Python/structmember.c index d5e7ab83093dc8..d36e049d6b5d20 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -34,40 +34,40 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) const char* addr = obj_addr + l->offset; switch (l->type) { case Py_T_BOOL: - v = PyBool_FromLong(*(char*)addr); + v = PyBool_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_BYTE: - v = PyLong_FromLong(*(char*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_CHAR_RELAXED(*(char*)addr)); break; case Py_T_UBYTE: - v = PyLong_FromUnsignedLong(*(unsigned char*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UCHAR_RELAXED(*(unsigned char*)addr)); break; case Py_T_SHORT: - v = PyLong_FromLong(*(short*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_SHORT_RELAXED(*(short*)addr)); break; case Py_T_USHORT: - v = PyLong_FromUnsignedLong(*(unsigned short*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_USHORT_RELAXED(*(unsigned short*)addr)); break; case Py_T_INT: - v = PyLong_FromLong(*(int*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_INT_RELAXED(*(int*)addr)); break; case Py_T_UINT: - v = PyLong_FromUnsignedLong(*(unsigned int*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_UINT_RELAXED(*(unsigned int*)addr)); break; case Py_T_LONG: - v = PyLong_FromLong(*(long*)addr); + v = PyLong_FromLong(FT_ATOMIC_LOAD_LONG_RELAXED(*(long*)addr)); break; case Py_T_ULONG: - v = PyLong_FromUnsignedLong(*(unsigned long*)addr); + v = PyLong_FromUnsignedLong(FT_ATOMIC_LOAD_ULONG_RELAXED(*(unsigned long*)addr)); break; case Py_T_PYSSIZET: - v = PyLong_FromSsize_t(*(Py_ssize_t*)addr); + v = PyLong_FromSsize_t(FT_ATOMIC_LOAD_SSIZE_RELAXED(*(Py_ssize_t*)addr)); break; case Py_T_FLOAT: - v = PyFloat_FromDouble((double)*(float*)addr); + v = PyFloat_FromDouble((double)FT_ATOMIC_LOAD_FLOAT_RELAXED(*(float*)addr)); break; case Py_T_DOUBLE: - v = PyFloat_FromDouble(*(double*)addr); + v = PyFloat_FromDouble(FT_ATOMIC_LOAD_DOUBLE_RELAXED(*(double*)addr)); break; case Py_T_STRING: if (*(char**)addr == NULL) { @@ -79,9 +79,11 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) case Py_T_STRING_INPLACE: v = PyUnicode_FromString((char*)addr); break; - case Py_T_CHAR: - v = PyUnicode_FromStringAndSize((char*)addr, 1); + case Py_T_CHAR: { + char char_val = FT_ATOMIC_LOAD_CHAR_RELAXED(*addr); + v = PyUnicode_FromStringAndSize(&char_val, 1); break; + } case _Py_T_OBJECT: v = *(PyObject **)addr; if (v == NULL) @@ -104,10 +106,10 @@ PyMember_GetOne(const char *obj_addr, PyMemberDef *l) #endif break; case Py_T_LONGLONG: - v = PyLong_FromLongLong(*(long long *)addr); + v = PyLong_FromLongLong(FT_ATOMIC_LOAD_LLONG_RELAXED(*(long long *)addr)); break; case Py_T_ULONGLONG: - v = PyLong_FromUnsignedLongLong(*(unsigned long long *)addr); + v = PyLong_FromUnsignedLongLong(FT_ATOMIC_LOAD_ULLONG_RELAXED(*(unsigned long long *)addr)); break; case _Py_T_NONE: // doesn't require free-threading code path @@ -169,16 +171,16 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) return -1; } if (v == Py_True) - *(char*)addr = (char) 1; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 1); else - *(char*)addr = (char) 0; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, 0); break; } case Py_T_BYTE:{ long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(char*)addr = (char)long_val; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, (char)long_val); /* XXX: For compatibility, only warn about truncations for now. */ if ((long_val > CHAR_MAX) || (long_val < CHAR_MIN)) @@ -189,7 +191,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned char*)addr = (unsigned char)long_val; + FT_ATOMIC_STORE_UCHAR_RELAXED(*(unsigned char*)addr, (unsigned char)long_val); if ((long_val > UCHAR_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned char"); break; @@ -198,7 +200,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(short*)addr = (short)long_val; + FT_ATOMIC_STORE_SHORT_RELAXED(*(short*)addr, (short)long_val); if ((long_val > SHRT_MAX) || (long_val < SHRT_MIN)) WARN("Truncation of value to short"); break; @@ -207,7 +209,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(unsigned short*)addr = (unsigned short)long_val; + FT_ATOMIC_STORE_USHORT_RELAXED(*(unsigned short*)addr, (unsigned short)long_val); if ((long_val > USHRT_MAX) || (long_val < 0)) WARN("Truncation of value to unsigned short"); break; @@ -216,7 +218,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) long long_val = PyLong_AsLong(v); if ((long_val == -1) && PyErr_Occurred()) return -1; - *(int *)addr = (int)long_val; + FT_ATOMIC_STORE_INT_RELAXED(*(int *)addr, (int)long_val); if ((long_val > INT_MAX) || (long_val < INT_MIN)) WARN("Truncation of value to int"); break; @@ -234,7 +236,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned int *)addr = (unsigned int)(unsigned long)long_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)(unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -243,19 +245,20 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned int*)addr = (unsigned int)ulong_val; + FT_ATOMIC_STORE_UINT_RELAXED(*(unsigned int *)addr, (unsigned int)ulong_val); if (ulong_val > UINT_MAX) { WARN("Truncation of value to unsigned int"); } } break; } - case Py_T_LONG:{ - *(long*)addr = PyLong_AsLong(v); - if ((*(long*)addr == -1) && PyErr_Occurred()) + case Py_T_LONG: { + const long long_val = PyLong_AsLong(v); + if ((long_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LONG_RELAXED(*(long*)addr, long_val); break; - } + } case Py_T_ULONG: { /* XXX: For compatibility, accept negative int values as well. */ @@ -269,7 +272,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long *)addr = (unsigned long)long_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, (unsigned long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -278,29 +281,31 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulong_val == (unsigned long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long*)addr = ulong_val; + FT_ATOMIC_STORE_ULONG_RELAXED(*(unsigned long *)addr, ulong_val); } break; } - case Py_T_PYSSIZET:{ - *(Py_ssize_t*)addr = PyLong_AsSsize_t(v); - if ((*(Py_ssize_t*)addr == (Py_ssize_t)-1) - && PyErr_Occurred()) - return -1; + case Py_T_PYSSIZET: { + const Py_ssize_t ssize_val = PyLong_AsSsize_t(v); + if ((ssize_val == (Py_ssize_t)-1) && PyErr_Occurred()) + return -1; + FT_ATOMIC_STORE_SSIZE_RELAXED(*(Py_ssize_t*)addr, ssize_val); break; - } + } case Py_T_FLOAT:{ double double_val = PyFloat_AsDouble(v); if ((double_val == -1) && PyErr_Occurred()) return -1; - *(float*)addr = (float)double_val; + FT_ATOMIC_STORE_FLOAT_RELAXED(*(float*)addr, (float)double_val); break; } - case Py_T_DOUBLE: - *(double*)addr = PyFloat_AsDouble(v); - if ((*(double*)addr == -1) && PyErr_Occurred()) + case Py_T_DOUBLE: { + const double double_val = PyFloat_AsDouble(v); + if ((double_val == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_DOUBLE_RELAXED(*(double *) addr, double_val); break; + } case _Py_T_OBJECT: case Py_T_OBJECT_EX: Py_BEGIN_CRITICAL_SECTION(obj); @@ -318,7 +323,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_BadArgument(); return -1; } - *(char*)addr = string[0]; + FT_ATOMIC_STORE_CHAR_RELAXED(*(char*)addr, string[0]); break; } case Py_T_STRING: @@ -326,10 +331,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_TypeError, "readonly attribute"); return -1; case Py_T_LONGLONG:{ - long long value; - *(long long*)addr = value = PyLong_AsLongLong(v); + long long value = PyLong_AsLongLong(v); if ((value == -1) && PyErr_Occurred()) return -1; + FT_ATOMIC_STORE_LLONG_RELAXED(*(long long*)addr, value); break; } case Py_T_ULONGLONG: { @@ -343,7 +348,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (long_val == -1 && PyErr_Occurred()) { return -1; } - *(unsigned long long *)addr = (unsigned long long)(long long)long_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, (unsigned long long)(long long)long_val); WARN("Writing negative value into unsigned field"); } else { @@ -352,7 +357,7 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) if (ulonglong_val == (unsigned long long)-1 && PyErr_Occurred()) { return -1; } - *(unsigned long long*)addr = ulonglong_val; + FT_ATOMIC_STORE_ULLONG_RELAXED(*(unsigned long long *)addr, ulonglong_val); } break; } diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index 16c38841e44a4d..d32747f19945d8 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'): # {AUTO_GENERATED_BY_SCRIPT} ''' token_py_template += ''' -__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF'] +__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF', + 'EXACT_TOKEN_TYPES'] %s N_TOKENS = %d diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 203286834e3e3f..d50c420307852f 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -309,7 +309,7 @@ This might become (if it was an instruction): ### More examples -For explanations see "Generating the interpreter" below.) +For explanations see "Generating the interpreter" below. ```C op ( CHECK_HAS_INSTANCE_VALUES, (owner -- owner) ) { PyDictOrValues dorv = *_PyObject_DictOrValuesPointer(owner); @@ -371,7 +371,7 @@ For explanations see "Generating the interpreter" below.) A _family_ maps a specializable instruction to its specializations. -Example: These opcodes all share the same instruction format): +Example: These opcodes all share the same instruction format: ```C family(load_attr) = { LOAD_ATTR, LOAD_ATTR_INSTANCE_VALUE, LOAD_SLOT }; ``` @@ -393,7 +393,7 @@ which can be easily inserted. What is more complex is ensuring the correct stack and not generating excess pops and pushes. For example, in `CHECK_HAS_INSTANCE_VALUES`, `owner` occurs in the input, so it cannot be -redefined. Thus it doesn't need to written and can be read without adjusting the stack pointer. +redefined. Thus it doesn't need to be written and can be read without adjusting the stack pointer. The C code generated for `CHECK_HAS_INSTANCE_VALUES` would look something like: ```C diff --git a/Tools/clinic/libclinic/parse_args.py b/Tools/clinic/libclinic/parse_args.py index fc2d9fe987096d..a57d729bec5733 100644 --- a/Tools/clinic/libclinic/parse_args.py +++ b/Tools/clinic/libclinic/parse_args.py @@ -146,7 +146,7 @@ def declare_parser( GETSET_DOCSTRING_PROTOTYPE_STRVAR: Final[str] = libclinic.normalize_snippet(""" PyDoc_STRVAR({getset_basename}__doc__, {docstring}); - #define {getset_basename}_HAS_DOCSTR + #define {getset_basename}_DOCSTR {getset_basename}__doc__ """) IMPL_DEFINITION_PROTOTYPE: Final[str] = libclinic.normalize_snippet(""" static {impl_return_type} @@ -157,9 +157,7 @@ def declare_parser( {{"{name}", {methoddef_cast}{c_basename}{methoddef_cast_end}, {methoddef_flags}, {c_basename}__doc__}}, """) GETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_basename}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else + #if !defined({getset_basename}_DOCSTR) # define {getset_basename}_DOCSTR NULL #endif #if defined({getset_name}_GETSETDEF) @@ -170,9 +168,7 @@ def declare_parser( #endif """) SETTERDEF_PROTOTYPE_DEFINE: Final[str] = libclinic.normalize_snippet(r""" - #if defined({getset_name}_HAS_DOCSTR) - # define {getset_basename}_DOCSTR {getset_basename}__doc__ - #else + #if !defined({getset_basename}_DOCSTR) # define {getset_basename}_DOCSTR NULL #endif #if defined({getset_name}_GETSETDEF) diff --git a/Tools/jit/ignore-tests-emulated-linux.txt b/Tools/jit/ignore-tests-emulated-linux.txt deleted file mode 100644 index 080a569574470c..00000000000000 --- a/Tools/jit/ignore-tests-emulated-linux.txt +++ /dev/null @@ -1,85 +0,0 @@ -test_multiprocessing_fork -test_strftime_y2k -test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_run -test.test_asyncio.test_unix_events.TestFork.test_fork_asyncio_subprocess -test.test_asyncio.test_unix_events.TestFork.test_fork_signal_handling -test.test_cmd_line.CmdLineTest.test_no_std_streams -test.test_cmd_line.CmdLineTest.test_no_stdin -test.test_concurrent_futures.test_init.ProcessPoolForkFailingInitializerTest.test_initializer -test.test_concurrent_futures.test_process_pool.ProcessPoolForkProcessPoolExecutorTest.test_ressources_gced_in_workers -test.test_external_inspection.TestGetStackTrace.test_remote_stack_trace -test.test_external_inspection.TestGetStackTrace.test_self_trace -test.test_faulthandler.FaultHandlerTests.test_enable_fd -test.test_faulthandler.FaultHandlerTests.test_enable_file -test.test_init.ProcessPoolForkFailingInitializerTest.test_initializer -test.test_logging.ConfigDictTest.test_111615 -test.test_logging.ConfigDictTest.test_config_queue_handler -test.test_logging.ConfigDictTest.test_multiprocessing_queues -test.test_logging.ConfigDictTest.test_config_queue_handler_multiprocessing_context -test.test_os.ForkTests.test_fork_warns_when_non_python_thread_exists -test.test_os.TimerfdTests.test_timerfd_initval -test.test_os.TimerfdTests.test_timerfd_interval -test.test_os.TimerfdTests.test_timerfd_TFD_TIMER_ABSTIME -test.test_pathlib.PathSubclassTest.test_is_mount_root -test.test_pathlib.PathTest.test_is_mount_root -test.test_pathlib.PosixPathTest.test_is_mount_root -test.test_pathlib.test_pathlib.PathSubclassTest.test_is_mount_root -test.test_pathlib.test_pathlib.PathTest.test_is_mount_root -test.test_pathlib.test_pathlib.PosixPathTest.test_is_mount_root -test.test_posix.TestPosixSpawn.test_close_file -test.test_posix.TestPosixSpawnP.test_close_file -test.test_posixpath.PosixPathTest.test_ismount -test.test_signal.StressTest.test_stress_modifying_handlers -test.test_socket.BasicCANTest.testFilter -test.test_socket.BasicCANTest.testLoopback -test.test_socket.LinuxKernelCryptoAPI.test_aead_aes_gcm -test.test_socket.LinuxKernelCryptoAPI.test_aes_cbc -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgIntoRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Minus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen0Plus1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen1 -test.test_socket.RecvmsgIntoSCMRightsStreamTest.testCmsgTruncLen2Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDP6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTrunc2Int -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSecondCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncInData -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgRFC3542AncillaryUDPLITE6Test.testSingleCmsgTruncLen0Minus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Minus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen0Plus1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen1 -test.test_socket.RecvmsgSCMRightsStreamTest.testCmsgTruncLen2Minus1 -test.test_subprocess.POSIXProcessTestCase.test_exception_bad_args_0 -test.test_subprocess.POSIXProcessTestCase.test_exception_bad_executable -test.test_subprocess.POSIXProcessTestCase.test_vfork_used_when_expected -test.test_subprocess.ProcessTestCase.test_cwd_with_relative_arg -test.test_subprocess.ProcessTestCase.test_cwd_with_relative_executable -test.test_subprocess.ProcessTestCase.test_empty_env -test.test_subprocess.ProcessTestCase.test_file_not_found_includes_filename -test.test_subprocess.ProcessTestCase.test_one_environment_variable -test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_arg -test.test_subprocess.ProcessTestCaseNoPoll.test_cwd_with_relative_executable -test.test_subprocess.ProcessTestCaseNoPoll.test_empty_env -test.test_subprocess.ProcessTestCaseNoPoll.test_file_not_found_includes_filename -test.test_subprocess.ProcessTestCaseNoPoll.test_one_environment_variable -test.test_venv.BasicTest.test_zippath_from_non_installed_posix diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 5793e5c649d6b3..bc7ccfe33e777d 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -483,7 +483,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoced because it it too short.", + "A potential trace is abandoned because it it too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" @@ -1118,6 +1118,8 @@ def calc_gc_stats(stats: Stats) -> Rows: Count(gen["collections"]), Count(gen["objects collected"]), Count(gen["object visits"]), + Count(gen["objects reachable from roots"]), + Count(gen["objects not reachable from roots"]), ) for (i, gen) in enumerate(gc_stats) ] @@ -1127,7 +1129,8 @@ def calc_gc_stats(stats: Stats) -> Rows: "GC collections and effectiveness", [ Table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + ("Generation:", "Collections:", "Objects collected:", "Object visits:", + "Reachable from roots:", "Not reachable from roots:"), calc_gc_stats, ) ], diff --git a/Tools/wasm/README.md b/Tools/wasm/README.md index 3f4211fb1dfb28..2e0fa2fb533d67 100644 --- a/Tools/wasm/README.md +++ b/Tools/wasm/README.md @@ -23,9 +23,9 @@ https://github.com/psf/webassembly for more information. To cross compile to the ``wasm32-emscripten`` platform you need [the Emscripten compiler toolchain](https://emscripten.org/), -a Python interpreter, and an installation of Node version 18 or newer. Emscripten -version 3.1.42 or newer is recommended. All commands below are relative to a checkout -of the Python repository. +a Python interpreter, and an installation of Node version 18 or newer. +Emscripten version 3.1.73 or newer is recommended. All commands below are +relative to a checkout of the Python repository. #### Install [the Emscripten compiler toolchain](https://emscripten.org/docs/getting_started/downloads.html) @@ -50,7 +50,7 @@ sourced. Otherwise the source script removes the environment variable. export EM_COMPILER_WRAPPER=ccache ``` -### Compile and build Python interpreter +#### Compile and build Python interpreter You can use `python Tools/wasm/emscripten` to compile and build targetting Emscripten. You can do everything at once with: @@ -61,6 +61,7 @@ or you can break it out into four separate steps: ```shell python Tools/wasm/emscripten configure-build-python python Tools/wasm/emscripten make-build-python +python Tools/wasm/emscripten make-libffi python Tools/wasm/emscripten configure-host python Tools/wasm/emscripten make-host ``` @@ -70,6 +71,88 @@ instance, to do a debug build, you can use: python Tools/wasm/emscripten build --with-py-debug ``` +### Running from node + +If you want to run the normal Python CLI, you can use `python.sh`. It takes the +same options as the normal Python CLI entrypoint, though the REPL does not +function and will crash. + +`python.sh` invokes `node_entry.mjs` which imports the Emscripten module for the +Python process and starts it up with the appropriate settings. If you wish to +make a node application that "embeds" the interpreter instead of acting like the +CLI you will need to write your own alternative to `node_entry.mjs`. + + +### The Web Example + +When building for Emscripten, the web example will be built automatically. It is +in the ``web_example`` directory. To run the web example, ``cd`` into the +``web_example`` directory, then run ``python server.py``. This will start a web +server; you can then visit ``http://localhost:8000/python.html`` in a browser to +see a simple REPL example. + +The web example relies on a bug fix in Emscripten version 3.1.73 so if you build +with earlier versions of Emscripten it may not work. The web example uses +``SharedArrayBuffer``. For security reasons browsers only provide +``SharedArrayBuffer`` in secure environments with cross-origin isolation. The +webserver must send cross-origin headers and correct MIME types for the +JavaScript and WebAssembly files. Otherwise the terminal will fail to load with +an error message like ``ReferenceError: SharedArrayBuffer is not defined``. See +more information here: +https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements + +Note that ``SharedArrayBuffer`` is _not required_ to use Python itself, only the +web example. If cross-origin isolation is not appropriate for your use case you +may make your own application embedding `python.mjs` which does not use +``SharedArrayBuffer`` and serve it without the cross-origin isolation headers. + +### Embedding Python in a custom JavaScript application + +You can look at `python.worker.mjs` and `node_entry.mjs` for inspiration. At a +minimum you must import ``createEmscriptenModule`` and you need to call +``createEmscriptenModule`` with an appropriate settings object. This settings +object will need a prerun hook that installs the Python standard library into +the Emscripten file system. + +#### NodeJs + +In Node, you can use the NodeFS to mount the standard library in your native +file system into the Emscripten file system: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + preRun(Module) { + Module.FS.mount( + Module.FS.filesystems.NODEFS, + { root: "/path/to/python/stdlib" }, + "/lib/", + ); + }, +}); +``` + +#### Browser + +In the browser, the simplest approach is to put the standard library in a zip +file it and install it. With Python 3.14 this could look like: +```js +import createEmscriptenModule from "./python.mjs"; + +await createEmscriptenModule({ + async preRun(Module) { + Module.FS.mkdirTree("/lib/python3.14/lib-dynload/"); + Module.addRunDependency("install-stdlib"); + const resp = await fetch("python3.14.zip"); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile(`/lib/python314.zip`, new Uint8Array(stdlibBuffer), { + canOwn: true, + }); + Module.removeRunDependency("install-stdlib"); + }, +}); +``` + ### Limitations and issues #### Network stack @@ -151,38 +234,6 @@ python Tools/wasm/emscripten build --with-py-debug - Test modules are disabled by default. Use ``--enable-test-modules`` build test modules like ``_testcapi``. -### wasm32-emscripten in node - -Node builds use ``NODERAWFS``. - -- Node RawFS allows direct access to the host file system without need to - perform ``FS.mount()`` call. - -### Hosting Python WASM builds - -The simple REPL terminal uses SharedArrayBuffer. For security reasons -browsers only provide the feature in secure environments with cross-origin -isolation. The webserver must send cross-origin headers and correct MIME types -for the JavaScript and WASM files. Otherwise the terminal will fail to load -with an error message like ``Browsers disable shared array buffer``. - -#### Apache HTTP .htaccess - -Place a ``.htaccess`` file in the same directory as ``python.wasm``. - -``` -# .htaccess -Header set Cross-Origin-Opener-Policy same-origin -Header set Cross-Origin-Embedder-Policy require-corp - -AddType application/javascript js -AddType application/wasm wasm - - - AddOutputFilterByType DEFLATE text/html application/javascript application/wasm - -``` - ## WASI (wasm32-wasi) See [the devguide on how to build and run for WASI](https://devguide.python.org/getting-started/setup-building/#wasi). diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 9ce8dd6a364ad6..4a53e0bd1bee1b 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -9,6 +9,7 @@ import sys import sysconfig import tempfile +from urllib.request import urlopen from pathlib import Path from textwrap import dedent @@ -22,9 +23,13 @@ CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" +NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" HOST_TRIPLE = "wasm32-emscripten" -HOST_DIR = CROSS_BUILD_DIR / HOST_TRIPLE + +DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" +HOST_DIR = HOST_BUILD_DIR / "python" +PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") @@ -118,16 +123,16 @@ def build_platform(): def build_python_path(): """The path to the build Python binary.""" - binary = BUILD_DIR / "python" + binary = NATIVE_BUILD_DIR / "python" if not binary.is_file(): binary = binary.with_suffix(".exe") if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{BUILD_DIR}") + raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") return binary -@subdir(BUILD_DIR, clean_ok=True) +@subdir(NATIVE_BUILD_DIR, clean_ok=True) def configure_build_python(context, working_dir): """Configure the build/host Python.""" if LOCAL_SETUP.exists(): @@ -143,7 +148,7 @@ def configure_build_python(context, working_dir): call(configure, quiet=context.quiet) -@subdir(BUILD_DIR) +@subdir(NATIVE_BUILD_DIR) def make_build_python(context, working_dir): """Make/build the build Python.""" call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) @@ -159,6 +164,23 @@ def make_build_python(context, working_dir): print(f"🎉 {binary} {version}") +@subdir(HOST_BUILD_DIR, clean_ok=True) +def make_emscripten_libffi(context, working_dir): + shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) + with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: + with urlopen( + "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" + ) as response: + shutil.copyfileobj(response, tmp_file) + shutil.unpack_archive(tmp_file.name, working_dir) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": PREFIX_DIR}), + cwd=working_dir / "libffi-3.4.6", + quiet=context.quiet, + ) + + @subdir(HOST_DIR, clean_ok=True) def configure_emscripten_python(context, working_dir): """Configure the emscripten/host build.""" @@ -168,7 +190,7 @@ def configure_emscripten_python(context, working_dir): emscripten_build_dir = working_dir.relative_to(CHECKOUT) - python_build_dir = BUILD_DIR / "build" + python_build_dir = NATIVE_BUILD_DIR / "build" lib_dirs = list(python_build_dir.glob("lib.*")) assert ( len(lib_dirs) == 1 @@ -183,12 +205,18 @@ def configure_emscripten_python(context, working_dir): sysconfig_data += "-pydebug" host_runner = context.host_runner - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } build_python = os.fsdecode(build_python_path()) configure = [ "emconfigure", os.path.relpath(CHECKOUT / "configure", working_dir), "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + "PKG_CONFIG=pkg-config", f"--host={HOST_TRIPLE}", f"--build={build_platform()}", f"--with-build-python={build_python}", @@ -197,7 +225,7 @@ def configure_emscripten_python(context, working_dir): "--disable-ipv6", "--enable-big-digits=30", "--enable-wasm-dynamic-linking", - f"--prefix={HOST_DIR}", + f"--prefix={PREFIX_DIR}", ] if pydebug: configure.append("--with-pydebug") @@ -218,9 +246,26 @@ def configure_emscripten_python(context, working_dir): f"""\ #!/bin/sh + # Macs come with FreeBSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null && realpath --version > /dev/null 2> /dev/null && realpath --version | grep GNU > /dev/null; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () {{ + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + }} + REALPATH=abs_path + fi + # We compute our own path, not following symlinks and pass it in so that # node_entry.mjs can set sys.executable correctly. - exec {host_runner} {node_entry} "$(realpath -s $0)" "$@" + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" """ ) ) @@ -233,7 +278,7 @@ def configure_emscripten_python(context, working_dir): def make_emscripten_python(context, working_dir): """Run `make` for the emscripten/host build.""" call( - ["make", "--jobs", str(cpu_count()), "commoninstall"], + ["make", "--jobs", str(cpu_count()), "all"], env=updated_env(), quiet=context.quiet, ) @@ -247,6 +292,7 @@ def build_all(context): steps = [ configure_build_python, make_build_python, + make_emscripten_libffi, configure_emscripten_python, make_emscripten_python, ] @@ -275,6 +321,9 @@ def main(): configure_build = subcommands.add_parser( "configure-build-python", help="Run `configure` for the " "build Python" ) + make_libffi_cmd = subcommands.add_parser( + "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + ) make_build = subcommands.add_parser( "make-build-python", help="Run `make` for the build Python" ) @@ -282,11 +331,20 @@ def main(): "configure-host", help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", ) - make_host = subcommands.add_parser("make-host", help="Run `make` for the host/emscripten") + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) clean = subcommands.add_parser( "clean", help="Delete files and directories created by this script" ) - for subcommand in build, configure_build, make_build, configure_host, make_host: + for subcommand in ( + build, + configure_build, + make_libffi_cmd, + make_build, + configure_host, + make_host, + ): subcommand.add_argument( "--quiet", action="store_true", @@ -319,6 +377,7 @@ def main(): context = parser.parse_args() dispatch = { + "make-libffi": make_emscripten_libffi, "configure-build-python": configure_build_python, "make-build-python": make_build_python, "configure-host": configure_emscripten_python, diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Tools/wasm/emscripten/make_libffi.sh new file mode 100755 index 00000000000000..3c75c4d5127102 --- /dev/null +++ b/Tools/wasm/emscripten/make_libffi.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set +e + +export CFLAGS="-O2 -fPIC -DWASM_BIGINT" +export CXXFLAGS="$CFLAGS" + +# Build paths +export CPATH="$PREFIX/include" +export PKG_CONFIG_PATH="$PREFIX/lib/pkgconfig" +export EM_PKG_CONFIG_PATH="$PKG_CONFIG_PATH" + +# Specific variables for cross-compilation +export CHOST="wasm32-unknown-linux" # wasm32-unknown-emscripten + +emconfigure ./configure --host=$CHOST --prefix="$PREFIX" --enable-static --disable-shared --disable-dependency-tracking \ + --disable-builddir --disable-multi-os-directory --disable-raw-api --disable-docs + +make install +# Some forgotten headers? +cp fficonfig.h $PREFIX/include/ +cp include/ffi_common.h $PREFIX/include/ diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Tools/wasm/emscripten/node_entry.mjs index cb1c6ff3cba6aa..98b8f572a7e762 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Tools/wasm/emscripten/node_entry.mjs @@ -1,30 +1,48 @@ import EmscriptenModule from "./python.mjs"; -import { dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; +import fs from "node:fs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); if (nodeVersion < 18) { - process.stderr.write( - `Node version must be >= 18, got version ${process.version}\n`, - ); - process.exit(1); + process.stderr.write( + `Node version must be >= 18, got version ${process.version}\n`, + ); + process.exit(1); } } +function rootDirsToMount(Module) { + return fs + .readdirSync("/") + .filter((dir) => !["dev", "lib", "proc"].includes(dir)) + .map((dir) => "/" + dir); +} + +function mountDirectories(Module) { + for (const dir of rootDirsToMount(Module)) { + Module.FS.mkdirTree(dir); + Module.FS.mount(Module.FS.filesystems.NODEFS, { root: dir }, dir); + } +} + +const thisProgram = "--this-program="; +const thisProgramIndex = process.argv.findIndex((x) => + x.startsWith(thisProgram), +); + const settings = { preRun(Module) { - const __dirname = dirname(fileURLToPath(import.meta.url)); - Module.FS.mkdirTree("/lib/"); - Module.FS.mount(Module.FS.filesystems.NODEFS, { root: __dirname + "/lib/" }, "/lib/"); + mountDirectories(Module); + Module.FS.chdir(process.cwd()); + Object.assign(Module.ENV, process.env); + delete Module.ENV.PATH; }, - // The first three arguments are: "node", path to this file, path to - // python.sh. After that come the arguments the user passed to python.sh. - arguments: process.argv.slice(3), // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. - thisProgram: process.argv[2], + thisProgram: process.argv[thisProgramIndex].slice(thisProgram.length), + // After python.sh come the arguments thatthe user passed to python.sh. + arguments: process.argv.slice(thisProgramIndex + 1), }; await EmscriptenModule(settings); diff --git a/Tools/wasm/emscripten/web_example/python.html b/Tools/wasm/emscripten/web_example/python.html new file mode 100644 index 00000000000000..078f86eb764419 --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.html @@ -0,0 +1,433 @@ + + + + + + + + + wasm-python terminal + + + + + +

Simple REPL for Python WASM

+ +
+ + + + +
+
+
+ The simple REPL provides a limited Python experience in the browser. + + Tools/wasm/README.md + + contains a list of known limitations and issues. Networking, + subprocesses, and threading are not available. +
+ + diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Tools/wasm/emscripten/web_example/python.worker.mjs new file mode 100644 index 00000000000000..8043e419966743 --- /dev/null +++ b/Tools/wasm/emscripten/web_example/python.worker.mjs @@ -0,0 +1,109 @@ +import createEmscriptenModule from "./python.mjs"; + +class StdinBuffer { + constructor() { + this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT); + this.buffer = new Int32Array(this.sab); + this.readIndex = 1; + this.numberOfCharacters = 0; + this.sentNull = true; + } + + prompt() { + this.readIndex = 1; + Atomics.store(this.buffer, 0, -1); + postMessage({ + type: "stdin", + buffer: this.sab, + }); + Atomics.wait(this.buffer, 0, -1); + this.numberOfCharacters = this.buffer[0]; + } + + stdin = () => { + while (this.numberOfCharacters + 1 === this.readIndex) { + if (!this.sentNull) { + // Must return null once to indicate we're done for now. + this.sentNull = true; + return null; + } + this.sentNull = false; + // Prompt will reset this.readIndex to 1 + this.prompt(); + } + const char = this.buffer[this.readIndex]; + this.readIndex += 1; + return char; + }; +} + +const stdout = (charCode) => { + if (charCode) { + postMessage({ + type: "stdout", + stdout: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; + +const stderr = (charCode) => { + if (charCode) { + postMessage({ + type: "stderr", + stderr: charCode, + }); + } else { + console.log(typeof charCode, charCode); + } +}; + +const stdinBuffer = new StdinBuffer(); + +const emscriptenSettings = { + noInitialRun: true, + stdin: stdinBuffer.stdin, + stdout: stdout, + stderr: stderr, + onRuntimeInitialized: () => { + postMessage({ type: "ready", stdinBuffer: stdinBuffer.sab }); + }, + async preRun(Module) { + const versionHex = Module.HEAPU32[Module._Py_Version / 4].toString(16); + const versionTuple = versionHex + .padStart(8, "0") + .match(/.{1,2}/g) + .map((x) => parseInt(x, 16)); + const [major, minor, ..._] = versionTuple; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + Module.addRunDependency("install-stdlib"); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); + Module.removeRunDependency("install-stdlib"); + }, +}; + +const modulePromise = createEmscriptenModule(emscriptenSettings); + +onmessage = async (event) => { + if (event.data.type === "run") { + const Module = await modulePromise; + if (event.data.files) { + for (const [filename, contents] of Object.entries(event.data.files)) { + Module.FS.writeFile(filename, contents); + } + } + const ret = Module.callMain(event.data.args); + postMessage({ + type: "finished", + returnCode: ret, + }); + } +}; diff --git a/Tools/wasm/wasm_webserver.py b/Tools/wasm/emscripten/web_example/server.py similarity index 85% rename from Tools/wasm/wasm_webserver.py rename to Tools/wasm/emscripten/web_example/server.py index 3d1d5d42a1e8c4..768e6f84e07798 100755 --- a/Tools/wasm/wasm_webserver.py +++ b/Tools/wasm/emscripten/web_example/server.py @@ -14,13 +14,6 @@ class MyHTTPRequestHandler(server.SimpleHTTPRequestHandler): - extensions_map = server.SimpleHTTPRequestHandler.extensions_map.copy() - extensions_map.update( - { - ".wasm": "application/wasm", - } - ) - def end_headers(self) -> None: self.send_my_headers() super().end_headers() @@ -42,5 +35,6 @@ def main() -> None: bind=args.bind, ) + if __name__ == "__main__": main() diff --git a/Tools/wasm/wasm_assets.py b/Tools/wasm/emscripten/web_example/wasm_assets.py similarity index 91% rename from Tools/wasm/wasm_assets.py rename to Tools/wasm/emscripten/web_example/wasm_assets.py index ffa5e303412c46..7f0fa7ae7c10ec 100755 --- a/Tools/wasm/wasm_assets.py +++ b/Tools/wasm/emscripten/web_example/wasm_assets.py @@ -19,7 +19,7 @@ from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parent.parent.parent.absolute() +SRCDIR = pathlib.Path(__file__).parents[4].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -28,9 +28,7 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = ( - WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" -) +WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -114,9 +112,7 @@ def get_sysconfigdata(args: argparse.Namespace) -> pathlib.Path: assert isinstance(args.builddir, pathlib.Path) data_name: str = sysconfig._get_sysconfigdata_name() # type: ignore[attr-defined] if not data_name.startswith(SYSCONFIG_NAMES): - raise ValueError( - f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES - ) + raise ValueError(f"Invalid sysconfig data name '{data_name}'.", SYSCONFIG_NAMES) filename = data_name + ".py" return args.builddir / filename @@ -131,7 +127,7 @@ def filterfunc(filename: str) -> bool: return pathname not in args.omit_files_absolute with zipfile.PyZipFile( - args.wasm_stdlib_zip, + args.output, mode="w", compression=args.compression, optimize=optimize, @@ -195,6 +191,12 @@ def path(val: str) -> pathlib.Path: default=pathlib.Path("/usr/local"), type=path, ) +parser.add_argument( + "-o", + "--output", + help="output file", + type=path, +) def main() -> None: @@ -204,7 +206,6 @@ def main() -> None: args.srcdir = SRCDIR args.srcdir_lib = SRCDIR_LIB args.wasm_root = args.buildroot / relative_prefix - args.wasm_stdlib_zip = args.wasm_root / WASM_STDLIB_ZIP args.wasm_stdlib = args.wasm_root / WASM_STDLIB args.wasm_dynload = args.wasm_root / WASM_DYNLOAD @@ -234,12 +235,10 @@ def main() -> None: args.wasm_dynload.mkdir(parents=True, exist_ok=True) marker = args.wasm_dynload / ".empty" marker.touch() - # os.py is a marker for finding the correct lib directory. - shutil.copy(args.srcdir_lib / "os.py", args.wasm_stdlib) # The rest of stdlib that's useful in a WASM context. create_stdlib_zip(args) - size = round(args.wasm_stdlib_zip.stat().st_size / 1024**2, 2) - parser.exit(0, f"Created {args.wasm_stdlib_zip} ({size} MiB)\n") + size = round(args.output.stat().st_size / 1024**2, 2) + parser.exit(0, f"Created {args.output} ({size} MiB)\n") if __name__ == "__main__": diff --git a/Tools/wasm/python.html b/Tools/wasm/python.html deleted file mode 100644 index 81a035a5c4cd93..00000000000000 --- a/Tools/wasm/python.html +++ /dev/null @@ -1,373 +0,0 @@ - - - - - - - - - wasm-python terminal - - - - - - -

Simple REPL for Python WASM

- -
- - - - -
-
-
- The simple REPL provides a limited Python experience in the browser. - - Tools/wasm/README.md contains a list of known limitations and - issues. Networking, subprocesses, and threading are not available. -
- - diff --git a/Tools/wasm/python.worker.js b/Tools/wasm/python.worker.js deleted file mode 100644 index 4ce4e16fc0fa19..00000000000000 --- a/Tools/wasm/python.worker.js +++ /dev/null @@ -1,87 +0,0 @@ -class StdinBuffer { - constructor() { - this.sab = new SharedArrayBuffer(128 * Int32Array.BYTES_PER_ELEMENT) - this.buffer = new Int32Array(this.sab) - this.readIndex = 1; - this.numberOfCharacters = 0; - this.sentNull = true - } - - prompt() { - this.readIndex = 1 - Atomics.store(this.buffer, 0, -1) - postMessage({ - type: 'stdin', - buffer: this.sab - }) - Atomics.wait(this.buffer, 0, -1) - this.numberOfCharacters = this.buffer[0] - } - - stdin = () => { - while (this.numberOfCharacters + 1 === this.readIndex) { - if (!this.sentNull) { - // Must return null once to indicate we're done for now. - this.sentNull = true - return null - } - this.sentNull = false - // Prompt will reset this.readIndex to 1 - this.prompt() - } - const char = this.buffer[this.readIndex] - this.readIndex += 1 - return char - } -} - -const stdout = (charCode) => { - if (charCode) { - postMessage({ - type: 'stdout', - stdout: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stderr = (charCode) => { - if (charCode) { - postMessage({ - type: 'stderr', - stderr: charCode, - }) - } else { - console.log(typeof charCode, charCode) - } -} - -const stdinBuffer = new StdinBuffer() - -var Module = { - noInitialRun: true, - stdin: stdinBuffer.stdin, - stdout: stdout, - stderr: stderr, - onRuntimeInitialized: () => { - postMessage({type: 'ready', stdinBuffer: stdinBuffer.sab}) - } -} - -onmessage = (event) => { - if (event.data.type === 'run') { - if (event.data.files) { - for (const [filename, contents] of Object.entries(event.data.files)) { - Module.FS.writeFile(filename, contents) - } - } - const ret = callMain(event.data.args) - postMessage({ - type: 'finished', - returnCode: ret - }) - } -} - -importScripts('python.js') diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index ac36d55587a38f..da847c4ff86215 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -297,8 +297,6 @@ def main(): # build. # Use 16 MiB stack. "--wasm max-wasm-stack=16777216 " - # Use WASI 0.2 primitives. - "--wasi preview2 " # Enable thread support; causes use of preview1. #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. diff --git a/configure b/configure index 84b74ac3584bcd..57be576e3cae99 100755 --- a/configure +++ b/configure @@ -821,6 +821,7 @@ MODULE_TIME_TRUE MODULE__IO_FALSE MODULE__IO_TRUE MODULE_BUILDTYPE +PYTHREAD_NAME_MAXLEN TEST_MODULES OPENSSL_LDFLAGS OPENSSL_LIBS @@ -944,8 +945,8 @@ AR LINK_PYTHON_OBJS LINK_PYTHON_DEPS LIBRARY_DEPS -NODE HOSTRUNNER +NODE STATIC_LIBPYTHON GNULD EXPORTSFROM @@ -1147,7 +1148,6 @@ LDFLAGS LIBS CPPFLAGS CPP -HOSTRUNNER PROFILE_TASK BOLT_INSTRUMENT_FLAGS BOLT_APPLY_FLAGS @@ -1968,7 +1968,6 @@ Some influential environment variables: CPPFLAGS (Objective) C/C++ preprocessor flags, e.g. -I if you have headers in a nonstandard directory CPP C preprocessor - HOSTRUNNER Program to run CPython for the host platform PROFILE_TASK Python args for PGO generation task BOLT_INSTRUMENT_FLAGS @@ -7622,9 +7621,9 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking HOSTRUNNER" >&5 printf %s "checking HOSTRUNNER... " >&6; } - if test -z "$HOSTRUNNER" then case $ac_sys_system in #( @@ -8335,8 +8334,12 @@ fi fi -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - DEF_MAKE_ALL_RULE="build_wasm" +elif test "$ac_sys_system" = "Emscripten"; then + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" else @@ -9427,12 +9430,14 @@ else $as_nop wasm_debug=no fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520" + as_fn_append LINKFORSHARED " -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520" as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" - as_fn_append LDFLAGS_NODIST " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LDFLAGS_NODIST " -sEXPORTED_RUNTIME_METHODS=FS" + as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version" + as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9449,7 +9454,6 @@ then : as_fn_append LINKFORSHARED " -sPROXY_TO_PTHREAD" fi - as_fn_append LDFLAGS_NODIST " -sALLOW_MEMORY_GROWTH" as_fn_append LDFLAGS_NODIST " -sEXIT_RUNTIME" WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" @@ -11175,6 +11179,12 @@ if test "x$ac_cv_header_sys_param_h" = xyes then : printf "%s\n" "#define HAVE_SYS_PARAM_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/pidfd.h" "ac_cv_header_sys_pidfd_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_pidfd_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_PIDFD_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/poll.h" "ac_cv_header_sys_poll_h" "$ac_includes_default" if test "x$ac_cv_header_sys_poll_h" = xyes @@ -18832,6 +18842,18 @@ if test "x$ac_cv_func_pthread_kill" = xyes then : printf "%s\n" "#define HAVE_PTHREAD_KILL 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pthread_getname_np" "ac_cv_func_pthread_getname_np" +if test "x$ac_cv_func_pthread_getname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_GETNAME_NP 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "pthread_setname_np" "ac_cv_func_pthread_setname_np" +if test "x$ac_cv_func_pthread_setname_np" = xyes +then : + printf "%s\n" "#define HAVE_PTHREAD_SETNAME_NP 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "ptsname" "ac_cv_func_ptsname" if test "x$ac_cv_func_ptsname" = xyes @@ -29072,6 +29094,23 @@ fi CPPFLAGS=$save_CPPFLAGS +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) PYTHREAD_NAME_MAXLEN=31;; + Darwin) PYTHREAD_NAME_MAXLEN=63;; + iOS) PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; + *) PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$PYTHREAD_NAME_MAXLEN"; then + +printf "%s\n" "#define PYTHREAD_NAME_MAXLEN $PYTHREAD_NAME_MAXLEN" >>confdefs.h + +fi + + + # stdlib diff --git a/configure.ac b/configure.ac index 8fa6cb60900ad1..bd0221481c5341 100644 --- a/configure.ac +++ b/configure.ac @@ -1609,8 +1609,8 @@ if test "$cross_compiling" = yes; then RUNSHARED= fi +# HOSTRUNNER - Program to run CPython for the host platform AC_MSG_CHECKING([HOSTRUNNER]) -AC_ARG_VAR([HOSTRUNNER], [Program to run CPython for the host platform]) if test -z "$HOSTRUNNER" then AS_CASE([$ac_sys_system], @@ -1854,9 +1854,13 @@ if test "$Py_OPT" = 'true' ; then LDFLAGS_NODIST="$LDFLAGS_NODIST -fno-semantic-interposition" ], [], [-Werror]) ]) -elif test "$ac_sys_system" = "Emscripten" -o "$ac_sys_system" = "WASI"; then - dnl Emscripten does not support shared extensions yet. Build - dnl "python.[js,wasm]", "pybuilddir.txt", and "platform" files. +elif test "$ac_sys_system" = "Emscripten"; then + dnl Build "python.[js,wasm]", "pybuilddir.txt", and "platform" files. + DEF_MAKE_ALL_RULE="build_emscripten" + REQUIRE_PGO="no" + DEF_MAKE_RULE="all" +elif test "$ac_sys_system" = "WASI"; then + dnl Build "python.wasm", "pybuilddir.txt", and "platform" files. DEF_MAKE_ALL_RULE="build_wasm" REQUIRE_PGO="no" DEF_MAKE_RULE="all" @@ -2321,14 +2325,16 @@ AS_CASE([$ac_sys_system], AS_VAR_IF([Py_DEBUG], [yes], [wasm_debug=yes], [wasm_debug=no]) dnl Start with 20 MB and allow to grow - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH -sTOTAL_MEMORY=20971520"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sALLOW_MEMORY_GROWTH -sINITIAL_MEMORY=20971520"]) dnl map int64_t and uint64_t to JS bigint AS_VAR_APPEND([LDFLAGS_NODIST], [" -sWASM_BIGINT"]) dnl Include file system support - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXPORTED_RUNTIME_METHODS=FS"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2339,7 +2345,6 @@ AS_CASE([$ac_sys_system], AS_VAR_APPEND([LDFLAGS_NODIST], [" -sUSE_PTHREADS"]) AS_VAR_APPEND([LINKFORSHARED], [" -sPROXY_TO_PTHREAD"]) ]) - AS_VAR_APPEND([LDFLAGS_NODIST], [" -sALLOW_MEMORY_GROWTH"]) dnl not completely sure whether or not we want -sEXIT_RUNTIME, keeping it for now. AS_VAR_APPEND([LDFLAGS_NODIST], [" -sEXIT_RUNTIME"]) WASM_LINKFORSHARED_DEBUG="-gseparate-dwarf --emit-symbol-map" @@ -2927,7 +2932,7 @@ AC_CHECK_HEADERS([ \ linux/tipc.h linux/wait.h netdb.h net/ethernet.h netinet/in.h netpacket/packet.h poll.h process.h pthread.h pty.h \ sched.h setjmp.h shadow.h signal.h spawn.h stropts.h sys/audioio.h sys/bsdtty.h sys/devpoll.h \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ - sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/poll.h \ + sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ @@ -5105,8 +5110,10 @@ AC_CHECK_FUNCS([ \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ - pread preadv preadv2 process_vm_readv pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ + pread preadv preadv2 process_vm_readv \ + pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ + pthread_kill pthread_getname_np pthread_setname_np \ + ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ sched_setparam sched_setscheduler sem_clockwait sem_getvalue sem_open \ sem_timedwait sem_unlink sendfile setegid seteuid setgid sethostname \ @@ -7493,6 +7500,22 @@ AS_VAR_IF([ac_cv_libatomic_needed], [yes], _RESTORE_VAR([CPPFLAGS]) +# gh-59705: Maximum length in bytes of a thread name +case "$ac_sys_system" in + Linux*) PYTHREAD_NAME_MAXLEN=15;; # Linux and Android + SunOS*) PYTHREAD_NAME_MAXLEN=31;; + Darwin) PYTHREAD_NAME_MAXLEN=63;; + iOS) PYTHREAD_NAME_MAXLEN=63;; + FreeBSD*) PYTHREAD_NAME_MAXLEN=98;; + *) PYTHREAD_NAME_MAXLEN=;; +esac +if test -n "$PYTHREAD_NAME_MAXLEN"; then + AC_DEFINE_UNQUOTED([PYTHREAD_NAME_MAXLEN], [$PYTHREAD_NAME_MAXLEN], + [Maximum length in bytes of a thread name]) +fi +AC_SUBST([PYTHREAD_NAME_MAXLEN]) + + # stdlib AC_DEFUN([PY_STDLIB_MOD_SET_NA], [ m4_foreach([mod], [$@], [ diff --git a/iOS/README.rst b/iOS/README.rst index e33455eef8f44a..13b885144932e4 100644 --- a/iOS/README.rst +++ b/iOS/README.rst @@ -285,52 +285,42 @@ This will: * Install the Python iOS framework into the copy of the testbed project; and * Run the test suite on an "iPhone SE (3rd generation)" simulator. -While the test suite is running, Xcode does not display any console output. -After showing some Xcode build commands, the console output will print ``Testing -started``, and then appear to stop. It will remain in this state until the test -suite completes. On a 2022 M1 MacBook Pro, the test suite takes approximately 12 -minutes to run; a couple of extra minutes is required to boot and prepare the -iOS simulator. - On success, the test suite will exit and report successful completion of the -test suite. No output of the Python test suite will be displayed. - -On failure, the output of the Python test suite *will* be displayed. This will -show the details of the tests that failed. +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. Debugging test failures ----------------------- -The easiest way to diagnose a single test failure is to open the testbed project -in Xcode and run the tests from there using the "Product > Test" menu item. - -To test in Xcode, you must ensure the testbed project has a copy of a compiled -framework. If you've configured your build with the default install location of -``iOS/Frameworks``, you can copy from that location into the test project. To -test on an ARM64 simulator, run:: - - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/arm64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +Running ``make test`` generates a standalone version of the ``iOS/testbed`` +project, and runs the full test suite. It does this using ``iOS/testbed`` +itself - the folder is an executable module that can be used to create and run +a clone of the testbed project. -To test on an x86-64 simulator, run:: +You can generate your own standalone testbed instance by running:: - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/* - $ cp -r iOS/Frameworks/x86_64-iphonesimulator/* iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator + $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed -To test on a physical device:: +This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the +path to the iOS simulator framework for your platform (ARM64 in this case); +``my-testbed`` is the name of the folder for the new testbed clone. - $ rm -rf iOS/testbed/Python.xcframework/ios-arm64/* - $ cp -r iOS/Frameworks/arm64-iphoneos/* iOS/testbed/Python.xcframework/ios-arm64 +You can then use the ``my-testbed`` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the ``os`` module, you might run:: -Alternatively, you can configure your build to install directly into the -testbed project. For a simulator, use:: + $ python my-testbed run -- test -W test_os - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator +This is the equivalent of running ``python -m test -W test_os`` on a desktop +Python build. Any arguments after the ``--`` will be passed to testbed as if +they were arguments to ``python -m`` on a desktop machine. -For a physical device, use:: +You can also open the testbed project in Xcode by running:: - --enable-framework=$(pwd)/iOS/testbed/Python.xcframework/ios-arm64 + $ open my-testbed/iOSTestbed.xcodeproj +This will allow you to use the full Xcode suite of tools for debugging. Testing on an iOS device ^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/iOS/Resources/bin/arm64-apple-ios-ar b/iOS/Resources/bin/arm64-apple-ios-ar index 8122332b9c1de0..3cf3eb218741fa 100755 --- a/iOS/Resources/bin/arm64-apple-ios-ar +++ b/iOS/Resources/bin/arm64-apple-ios-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang b/iOS/Resources/bin/arm64-apple-ios-clang index 4d525751eba798..c39519cd1f8c94 100755 --- a/iOS/Resources/bin/arm64-apple-ios-clang +++ b/iOS/Resources/bin/arm64-apple-ios-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-clang++ b/iOS/Resources/bin/arm64-apple-ios-clang++ index f24bec11268f7e..d9b12925f384b9 100755 --- a/iOS/Resources/bin/arm64-apple-ios-clang++ +++ b/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-cpp b/iOS/Resources/bin/arm64-apple-ios-cpp index 891bb25bb4318c..24da23d3448ae0 100755 --- a/iOS/Resources/bin/arm64-apple-ios-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E $@ +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios -E "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/iOS/Resources/bin/arm64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/iOS/Resources/bin/arm64-apple-ios-simulator-clang index 32574cad284441..92e8d853d6ebc3 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ index ef37d05b512959..076469cc70cf98 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp index 6aaf6fbe188c32..c57f28cee5bcfe 100755 --- a/iOS/Resources/bin/arm64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios-simulator -E "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar index 74ed3bc6df1c2b..b836b6db9025bb 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-ar +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang index bcbe91f6061e16..17cbe0c8a1e213 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ index 86f03ea32bc2fd..565d47b24c214b 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios-simulator "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp index e6a42d9b85dec7..63fc8e8de2d38d 100755 --- a/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -1,2 +1,2 @@ #!/bin/sh -xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E $@ +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios-simulator -E "$@" diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py new file mode 100644 index 00000000000000..22570ee0f3ed04 --- /dev/null +++ b/iOS/testbed/__main__.py @@ -0,0 +1,365 @@ +import argparse +import asyncio +import json +import plistlib +import shutil +import subprocess +import sys +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path + + +DECODE_ARGS = ("UTF-8", "backslashreplace") + + +# Work around a bug involving sys.exit and TaskGroups +# (https://github.com/python/cpython/issues/101515). +def exit(*args): + raise MySystemExit(*args) + + +class MySystemExit(Exception): + pass + + +# All subprocesses are executed through this context manager so that no matter +# what happens, they can always be cancelled from another task, and they will +# always be cleaned up on exit. +@asynccontextmanager +async def async_process(*args, **kwargs): + process = await asyncio.create_subprocess_exec(*args, **kwargs) + try: + yield process + finally: + if process.returncode is None: + # Allow a reasonably long time for Xcode to clean itself up, + # because we don't want stale emulators left behind. + timeout = 10 + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout) + except TimeoutError: + print( + f"Command {args} did not terminate after {timeout} seconds " + f" - sending SIGKILL" + ) + process.kill() + + # Even after killing the process we must still wait for it, + # otherwise we'll get the warning "Exception ignored in __del__". + await asyncio.wait_for(process.wait(), timeout=1) + + +async def async_check_output(*args, **kwargs): + async with async_process( + *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs + ) as process: + stdout, stderr = await process.communicate() + if process.returncode == 0: + return stdout.decode(*DECODE_ARGS) + else: + raise subprocess.CalledProcessError( + process.returncode, + args, + stdout.decode(*DECODE_ARGS), + stderr.decode(*DECODE_ARGS), + ) + + +# Return a list of UDIDs associated with booted simulators +async def list_devices(): + # List the testing simulators, in JSON format + raw_json = await async_check_output( + "xcrun", "simctl", "--set", "testing", "list", "-j" + ) + json_data = json.loads(raw_json) + + # Filter out the booted iOS simulators + return [ + simulator["udid"] + for runtime, simulators in json_data["devices"].items() + for simulator in simulators + if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" + ] + + +async def find_device(initial_devices): + while True: + new_devices = set(await list_devices()).difference(initial_devices) + if len(new_devices) == 0: + await asyncio.sleep(1) + elif len(new_devices) == 1: + udid = new_devices.pop() + print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") + print(f"UDID: {udid}") + return udid + else: + exit(f"Found more than one new device: {new_devices}") + + +async def log_stream_task(initial_devices): + # Wait up to 5 minutes for the build to complete and the simulator to boot. + udid = await asyncio.wait_for(find_device(initial_devices), 5 * 60) + + # Stream the iOS device's logs, filtering out messages that come from the + # XCTest test suite (catching NSLog messages from the test method), or + # Python itself (catching stdout/stderr content routed to the system log + # with config->use_system_logger). + args = [ + "xcrun", + "simctl", + "--set", + "testing", + "spawn", + udid, + "log", + "stream", + "--style", + "compact", + "--predicate", + ( + 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' + ' OR senderImagePath ENDSWITH "/Python.framework/Python"' + ), + ] + + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + suppress_dupes = False + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + # The iOS log streamer can sometimes lag; when it does, it outputs + # a warning about messages being dropped... often multiple times. + # Only print the first of these duplicated warnings. + if line.startswith("=== Messages dropped "): + if not suppress_dupes: + suppress_dupes = True + sys.stdout.write(line) + else: + suppress_dupes = False + sys.stdout.write(line) + + +async def xcode_test(location, simulator): + # Run the test suite on the named simulator + args = [ + "xcodebuild", + "test", + "-project", + str(location / "iOSTestbed.xcodeproj"), + "-scheme", + "iOSTestbed", + "-destination", + f"platform=iOS Simulator,name={simulator}", + "-resultBundlePath", + str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), + "-derivedDataPath", + str(location / "DerivedData"), + ] + async with async_process( + *args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) as process: + while line := (await process.stdout.readline()).decode(*DECODE_ARGS): + sys.stdout.write(line) + + status = await asyncio.wait_for(process.wait(), timeout=1) + exit(status) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not (source / "Python.xcframework/ios-arm64_x86_64-simulator/bin").is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + f"a simulator framework. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project...") + shutil.copytree(source, target) + + if framework is not None: + if framework.suffix == ".xcframework": + print("Installing XCFramework...") + xc_framework_path = target / "Python.xcframework" + shutil.rmtree(xc_framework_path) + shutil.copytree(framework, xc_framework_path) + else: + print("Installing simulator Framework...") + sim_framework_path = ( + target / "Python.xcframework" / "ios-arm64_x86_64-simulator" + ) + shutil.rmtree(sim_framework_path) + shutil.copytree(framework, sim_framework_path) + else: + print("Using pre-existing iOS framework.") + + for app_src in apps: + print(f"Installing app {app_src.name!r}...") + app_target = target / f"iOSTestbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + + print(f"Testbed project created in {target}") + + +def update_plist(testbed_path, args): + # Add the test runner arguments to the testbed's Info.plist file. + info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" + with info_plist.open("rb") as f: + info = plistlib.load(f) + + info["TestArgs"] = args + + with info_plist.open("wb") as f: + plistlib.dump(info, f) + + +async def run_testbed(simulator: str, args: list[str]): + location = Path(__file__).parent + print("Updating plist...") + update_plist(location, args) + + # Get the list of devices that are booted at the start of the test run. + # The simulator started by the test suite will be detected as the new + # entry that appears on the device list. + initial_devices = await list_devices() + + try: + async with asyncio.TaskGroup() as tg: + tg.create_task(log_stream_task(initial_devices)) + tg.create_task(xcode_test(location, simulator)) + except* MySystemExit as e: + raise SystemExit(*e.exceptions[0].args) from None + except* subprocess.CalledProcessError as e: + # Extract it from the ExceptionGroup so it can be handled by `main`. + raise e.exceptions[0] + + +def main(): + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing a Python project in the iOS simulator." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in an iOS Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage="%(prog)s [-h] [--simulator SIMULATOR] -- [ ...]", + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--simulator", + default="iPhone SE (3rd Generation)", + help="The name of the simulator to use (default: 'iPhone SE (3rd Generation)')", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent, + target=Path(context.location), + framework=Path(context.framework) if context.framework else None, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" + ).is_dir(): + print( + f"Testbed does not contain a compiled iOS framework. Use " + f"`python {sys.argv[0]} clone ...` to create a runnable " + f"clone of this testbed." + ) + sys.exit(20) + + asyncio.run( + run_testbed( + simulator=context.simulator, + args=test_args, + ) + ) + else: + print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj index 6819ac0eeed95f..c7d63909ee2453 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -263,6 +263,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; + showEnvVarsInLog = 0; }; 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { isa = PBXShellScriptBuildPhase; @@ -282,6 +283,7 @@ runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m index db00d43da85cbc..ac78456a61e65e 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m @@ -50,6 +50,8 @@ - (void)testPython { // Enforce UTF-8 encoding for stderr, stdout, file-system encoding and locale. // See https://docs.python.org/3/library/os.html#python-utf-8-mode. preconfig.utf8_mode = 1; + // Use the system logger for stdout/err + config.use_system_logger = 1; // Don't buffer stdio. We want output to appears in the log immediately config.buffered_stdio = 0; // Don't write bytecode; we can't modify the app bundle diff --git a/pyconfig.h.in b/pyconfig.h.in index 924d86627b0e9b..166c195a8c66fc 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -981,6 +981,9 @@ /* Define to 1 if you have the `pthread_getcpuclockid' function. */ #undef HAVE_PTHREAD_GETCPUCLOCKID +/* Define to 1 if you have the `pthread_getname_np' function. */ +#undef HAVE_PTHREAD_GETNAME_NP + /* Define to 1 if you have the header file. */ #undef HAVE_PTHREAD_H @@ -990,6 +993,9 @@ /* Define to 1 if you have the `pthread_kill' function. */ #undef HAVE_PTHREAD_KILL +/* Define to 1 if you have the `pthread_setname_np' function. */ +#undef HAVE_PTHREAD_SETNAME_NP + /* Define to 1 if you have the `pthread_sigmask' function. */ #undef HAVE_PTHREAD_SIGMASK @@ -1388,6 +1394,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_SYS_PARAM_H +/* Define to 1 if you have the header file. */ +#undef HAVE_SYS_PIDFD_H + /* Define to 1 if you have the header file. */ #undef HAVE_SYS_POLL_H @@ -1647,6 +1656,9 @@ /* Define as the preferred size in bits of long digits */ #undef PYLONG_BITS_IN_DIGIT +/* Maximum length in bytes of a thread name */ +#undef PYTHREAD_NAME_MAXLEN + /* enabled builtin hash modules */ #undef PY_BUILTIN_HASHLIB_HASHES