diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 95b667ac946..60744cadbf7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,9 @@ on: MSRV: required: true type: string + verbose: + type: boolean + default: false jobs: build: @@ -27,6 +30,15 @@ jobs: if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: - uses: actions/checkout@v4 + with: + # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch. + # + # This is necessary for coverage reporting to make sense; we then get exactly the coverage change + # between the base branch and the real PR head. + # + # If it were run on the merge commit the problem is that the coverage potentially does not align + # with the commit diff, because the merge may affect line numbers. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v5 @@ -40,17 +52,26 @@ jobs: - name: Install nox run: python -m pip install --upgrade pip && pip install nox - - if: inputs.python-version == 'graalpy24.1' - name: Install GraalPy virtualenv (only GraalPy 24.1) - run: python -m pip install 'git+https://github.com/oracle/graalpython#egg=graalpy_virtualenv_seeder&subdirectory=graalpy_virtualenv_seeder' - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: toolchain: ${{ inputs.rust }} targets: ${{ inputs.rust-target }} - # needed to correctly format errors, see #1865 - components: rust-src + # rust-src needed to correctly format errors, see #1865 + components: rust-src,llvm-tools-preview + + # On windows 32 bit, we are running on an x64 host, so we need to specifically set the target + # NB we don't do this for *all* jobs because it breaks coverage of proc macros to have an + # explicit target set. + - name: Set Rust target for Windows 32-bit + if: inputs.os == 'windows-latest' && inputs.python-architecture == 'x86' + shell: bash + run: | + echo "CARGO_BUILD_TARGET=i686-pc-windows-msvc" >> $GITHUB_ENV + + - name: Install zoneinfo backport for Python 3.7 / 3.8 + if: contains(fromJSON('["3.7", "3.8"]'), inputs.python-version) + run: python -m pip install backports.zoneinfo - uses: Swatinem/rust-cache@v2 with: @@ -68,80 +89,12 @@ jobs: name: Ignore changed error messages when using trybuild run: echo "TRYBUILD=overwrite" >> "$GITHUB_ENV" - - if: inputs.rust == 'nightly' - name: Prepare to test on nightly rust - run: echo "MAYBE_NIGHTLY=nightly" >> "$GITHUB_ENV" - - - name: Build docs - run: nox -s docs - - - name: Build (no features) - if: ${{ !startsWith(inputs.python-version, 'graalpy') }} - run: cargo build --lib --tests --no-default-features - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Build pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo build --no-default-features - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test (no features) - run: cargo test --no-default-features --lib --tests - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Test pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo test --no-default-features - - - name: Build (all additive features) - if: ${{ !startsWith(inputs.python-version, 'graalpy') }} - run: cargo build --lib --tests --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" - - - if: ${{ startsWith(inputs.python-version, 'pypy') }} - name: Build PyPy (abi3-py39) - run: cargo build --lib --tests --no-default-features --features "multiple-pymethods abi3-py39 full $MAYBE_NIGHTLY" - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test - run: cargo test --no-default-features --features "full $MAYBE_NIGHTLY" - - # Repeat, with multiple-pymethods feature enabled (it's not entirely additive) - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test - run: cargo test --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" - - # Run tests again, but in abi3 mode - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test (abi3) - run: cargo test --no-default-features --features "multiple-pymethods abi3 full $MAYBE_NIGHTLY" - - # Run tests again, for abi3-py37 (the minimal Python version) - - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }} - name: Test (abi3-py37) - run: cargo test --no-default-features --features "multiple-pymethods abi3-py37 full $MAYBE_NIGHTLY" - - - name: Test proc-macro code - run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml - - - name: Test build config - run: cargo test --manifest-path=pyo3-build-config/Cargo.toml - - - name: Test python examples and tests - shell: bash - run: nox -s test-py - env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target - - uses: dorny/paths-filter@v3 if: ${{ inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') }} id: ffi-changes with: - base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} + base: ${{ github.event.merge_group.base_ref }} + ref: ${{ github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' @@ -154,9 +107,49 @@ jobs: if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check + - if: ${{ github.event_name != 'merge_group' }} + name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - if: ${{ github.event_name != 'merge_group' }} + name: Prepare coverage environment + run: | + cargo llvm-cov clean --workspace --profraw-only + nox -s set-coverage-env + + - name: Build docs + run: nox -s docs + + - name: Run Rust tests + run: nox -s test-rust + + - name: Test python examples and tests + shell: bash + run: nox -s test-py + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') }} + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + + - if: ${{ github.event_name != 'merge_group' }} + name: Generate coverage report + run: cargo llvm-cov + --package=pyo3 + --package=pyo3-build-config + --package=pyo3-macros-backend + --package=pyo3-macros + --package=pyo3-ffi + report --codecov --output-path coverage.json + + - if: ${{ github.event_name != 'merge_group' }} + name: Upload coverage report + uses: codecov/codecov-action@v5 + with: + files: coverage.json + name: ${{ inputs.os }}/${{ inputs.python-version }}/${{ inputs.rust }} + token: ${{ secrets.CODECOV_TOKEN }} + env: - CARGO_TERM_VERBOSE: true - CARGO_BUILD_TARGET: ${{ inputs.rust-target }} + CARGO_TERM_VERBOSE: ${{ inputs.verbose }} RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 227dcdc3825..428b78e5c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: runs-on: ubuntu-latest outputs: MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + verbose: ${{ runner.debug == '1' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -175,6 +176,7 @@ jobs: rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present @@ -225,6 +227,17 @@ jobs: python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } + # Also test free-threaded Python just for latest Python version, on ubuntu + # (run for all OSes on build-full) + - rust: stable + python-version: "3.13t" + platform: + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + build-full: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} @@ -237,6 +250,7 @@ jobs: rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present @@ -252,11 +266,13 @@ jobs: "3.11", "3.12", "3.13", + "3.13t", + "3.14-dev", + "3.14t-dev", "pypy3.9", "pypy3.10", "pypy3.11", - "graalpy24.0", - "graalpy24.1", + "graalpy24.2", ] platform: [ @@ -460,35 +476,6 @@ jobs: components: rust-src - run: cargo rustdoc --lib --no-default-features --features full,jiff-02 -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" - coverage: - if: ${{ github.event_name != 'merge_group' }} - needs: [fmt] - name: coverage ${{ matrix.os }} - strategy: - matrix: - os: ["windows-latest", "macos-latest", "ubuntu-latest"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - - uses: dtolnay/rust-toolchain@stable - with: - components: llvm-tools-preview,rust-src - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - run: python -m pip install --upgrade pip && pip install nox - - run: nox -s coverage - - uses: codecov/codecov-action@v5 - with: - files: coverage.json - name: ${{ matrix.os }} - token: ${{ secrets.CODECOV_TOKEN }} - emscripten: name: emscripten if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -545,8 +532,8 @@ jobs: components: rust-src - name: Install python3 standalone debug build with nox run: | - PBS_RELEASE="20241016" - PBS_PYTHON_VERSION="3.13.0" + PBS_RELEASE="20241219" + PBS_PYTHON_VERSION="3.13.1" PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst" wget "https://github.com/indygreg/python-build-standalone/releases/download/${PBS_RELEASE}/${PBS_ARCHIVE}" tar -I zstd -xf "${PBS_ARCHIVE}" @@ -575,43 +562,6 @@ jobs: echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV - run: python3 -m nox -s test - test-free-threaded: - needs: [fmt] - name: Free threaded tests - ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - - uses: dtolnay/rust-toolchain@stable - with: - components: rust-src - - uses: actions/setup-python@v5.5.0 - with: - python-version: "3.13t" - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - run: python3 -m sysconfig - - run: python3 -m pip install --upgrade pip && pip install nox - - name: Prepare coverage environment - run: | - cargo llvm-cov clean --workspace --profraw-only - nox -s set-coverage-env - - run: nox -s ffi-check - - run: nox - - name: Generate coverage report - run: nox -s generate-coverage-report - - name: Upload coverage report - uses: codecov/codecov-action@v5 - with: - files: coverage.json - name: ${{ matrix.os }}-test-free-threaded - token: ${{ secrets.CODECOV_TOKEN }} - test-version-limits: needs: [fmt] if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -758,30 +708,32 @@ jobs: test-introspection: needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-test-introspection') || contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} strategy: matrix: - platform: [ - { - os: "macos-latest", - python-architecture: "arm64", - rust-target: "aarch64-apple-darwin", - }, - { - os: "ubuntu-latest", - python-architecture: "x64", - rust-target: "x86_64-unknown-linux-gnu", - }, - { - os: "windows-latest", - python-architecture: "x64", - rust-target: "x86_64-pc-windows-msvc", - }, - { - os: "windows-latest", - python-architecture: "x86", - rust-target: "i686-pc-windows-msvc", - }, - ] + platform: + [ + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + }, + { + os: "windows-latest", + python-architecture: "x64", + rust-target: "x86_64-pc-windows-msvc", + }, + { + os: "windows-latest", + python-architecture: "x86", + rust-target: "i686-pc-windows-msvc", + }, + ] runs-on: ${{ matrix.platform.os }} steps: - uses: actions/checkout@v4 @@ -811,10 +763,8 @@ jobs: - valgrind - careful - docsrs - - coverage - emscripten - test-debug - - test-free-threaded - test-version-limits - check-feature-powerset - test-cross-compilation diff --git a/CHANGELOG.md b/CHANGELOG.md index 1532d3a7e9f..b13c92caef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,91 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.25.0] - 2025-05-14 + +### Packaging + +- Support Python 3.14.0b1. [#4811](https://github.com/PyO3/pyo3/pull/4811) +- Bump supported GraalPy version to 24.2. [#5116](https://github.com/PyO3/pyo3/pull/5116) +- Add optional `bigdecimal` dependency to add conversions for `bigdecimal::BigDecimal`. [#5011](https://github.com/PyO3/pyo3/pull/5011) +- Add optional `time` dependency to add conversions for `time` types. [#5057](https://github.com/PyO3/pyo3/pull/5057) +- Remove `cfg-if` dependency. [#5110](https://github.com/PyO3/pyo3/pull/5110) +- Add optional `ordered_float` dependency to add conversions for `ordered_float::NotNan` and `ordered_float::OrderedFloat`. [#5114](https://github.com/PyO3/pyo3/pull/5114) + +### Added + +- Add initial type stub generation to the `experimental-inspect` feature. [#3977](https://github.com/PyO3/pyo3/pull/3977) +- Add `#[pyclass(generic)]` option to support runtime generic typing. [#4926](https://github.com/PyO3/pyo3/pull/4926) +- Implement `OnceExt` & `MutexExt` for `parking_lot` & `lock_api`. Use the new extension traits by enabling the `arc_lock`, `lock_api`, or `parking_lot` cargo features. [#5044](https://github.com/PyO3/pyo3/pull/5044) +- Implement `From`/`Into` for `Borrowed` -> `Py`. [#5054](https://github.com/PyO3/pyo3/pull/5054) +- Add `PyTzInfo` constructors. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Add FFI definition `PY_INVALID_STACK_EFFECT`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Implement `AsRef>` for `Py`, `Bound` and `Borrowed`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Add FFI definition `PyModule_Add` and `compat::PyModule_Add`. [#5085](https://github.com/PyO3/pyo3/pull/5085) +- Add FFI definitions `Py_HashBuffer`, `Py_HashPointer`, and `PyObject_GenericHash`. [#5086](https://github.com/PyO3/pyo3/pull/5086) +- Support `#[pymodule_export]` on `const` items in declarative modules. [#5096](https://github.com/PyO3/pyo3/pull/5096) +- Add `#[pyclass(immutable_type)]` option (on Python 3.14+ with `abi3`, or 3.10+ otherwise) for immutable type objects. [#5101](https://github.com/PyO3/pyo3/pull/5101) +- Support `#[pyo3(rename_all)]` support on `#[derive(IntoPyObject)]`. [#5112](https://github.com/PyO3/pyo3/pull/5112) +- Add `PyRange` wrapper. [#5117](https://github.com/PyO3/pyo3/pull/5117) + +### Changed + +- Enable use of `datetime` types with `abi3` feature enabled. [#4970](https://github.com/PyO3/pyo3/pull/4970) +- Deprecate `timezone_utc` in favor of `PyTzInfo::utc`. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Reduce visibility of some CPython implementation details: [#5064](https://github.com/PyO3/pyo3/pull/5064) + - The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. + - The FFI definition `PyFutureFeatures` is now only defined up until Python 3.10 (it was present in CPython headers but unused in 3.11 and 3.12). +- Change `PyAnyMethods::is` to take `other: &Bound`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `Py::is` to take `other: &Py`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `PyVisit::call` to take `T: Into>>`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Expose `PyDateTime_DATE_GET_TZINFO` and `PyDateTime_TIME_GET_TZINFO` on PyPy 3.10 and later. [#5079](https://github.com/PyO3/pyo3/pull/5079) +- Add `#[track_caller]` to `with_gil` and `with_gil_unchecked`. [#5109](https://github.com/PyO3/pyo3/pull/5109) +- Use `std::thread::park()` instead of `libc::pause()` or `sleep(9999999)`. [#5115](https://github.com/PyO3/pyo3/pull/5115) + +### Removed + +- Remove all functionality deprecated in PyO3 0.23. [#4982](https://github.com/PyO3/pyo3/pull/4982) +- Remove deprecated `IntoPy` and `ToPyObject` traits. [#5010](https://github.com/PyO3/pyo3/pull/5010) +- Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove FFI definition `PyCode_GetNumFree` (PyO3 cannot support it due to knowledge of the code object). [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove `AsPyPointer` trait. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Remove support for the deprecated string form of `from_py_with`. [#5097](https://github.com/PyO3/pyo3/pull/5097) +- Remove FFI definitions of private static variables: `_PyMethodWrapper_Type`, `_PyCoroWrapper_Type`, `_PyImport_FrozenBootstrap`, `_PyImport_FrozenStdlib`, `_PyImport_FrozenTest`, `_PyManagedBuffer_Type`, `_PySet_Dummy`, `_PyWeakref_ProxyType`, and `_PyWeakref_CallableProxyType`. [#5105](https://github.com/PyO3/pyo3/pull/5105) +- Remove FFI definitions `PyASCIIObjectState`, `PyUnicode_IS_ASCII`, `PyUnicode_IS_COMPACT`, and `PyUnicode_IS_COMPACT_ASCII` on Python 3.14 and newer. [#5133](https://github.com/PyO3/pyo3/pull/5133) + +### Fixed + +- Correctly pick up the shared state for conda-based Python installation when reading information from sysconfigdata. [#5037](https://github.com/PyO3/pyo3/pull/5037) +- Fix compile failure with `#[derive(IntoPyObject, FromPyObject)]` when using `#[pyo3()]` options recognised by only one of the two derives. [#5070](https://github.com/PyO3/pyo3/pull/5070) +- Fix various compile errors from missing FFI definitions using certain feature combinations on PyPy and GraalPy. [#5091](https://github.com/PyO3/pyo3/pull/5091) +- Fallback on `backports.zoneinfo` for python <3.9 when converting timezones into python. [#5120](https://github.com/PyO3/pyo3/pull/5120) + +## [0.24.2] - 2025-04-21 + +### Fixed + +- Fix `unused_imports` lint of `#[pyfunction]` and `#[pymethods]` expanded in `macro_rules` context. [#5030](https://github.com/PyO3/pyo3/pull/5030) +- Fix size of `PyCodeObject::_co_instrumentation_version` ffi struct member on Python 3.13 for systems where `uintptr_t` is not 64 bits. [#5048](https://github.com/PyO3/pyo3/pull/5048) +- Fix struct-type complex enum variant fields incorrectly exposing raw identifiers as `r#ident` in Python bindings. [#5050](https://github.com/PyO3/pyo3/pull/5050) + +## [0.24.1] - 2025-03-31 + +### Added + +- Add `abi3-py313` feature. [#4969](https://github.com/PyO3/pyo3/pull/4969) +- Add `PyAnyMethods::getattr_opt`. [#4978](https://github.com/PyO3/pyo3/pull/4978) +- Add `PyInt::new` constructor for all supported number types (i32, u32, i64, u64, isize, usize). [#4984](https://github.com/PyO3/pyo3/pull/4984) +- Add `pyo3::sync::with_critical_section2`. [#4992](https://github.com/PyO3/pyo3/pull/4992) +- Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. [#5013](https://github.com/PyO3/pyo3/pull/5013) + +### Fixed + +- Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. [#4981](https://github.com/PyO3/pyo3/pull/4981) +- Fix `Probe` class naming issue with `#[pymethods]`. [#4988](https://github.com/PyO3/pyo3/pull/4988) +- Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). [#5002](https://github.com/PyO3/pyo3/pull/5002) +- Fix `PyString::from_object` causing of bounds reads whith `encoding` and `errors` parameters which are not nul-terminated. [#5008](https://github.com/PyO3/pyo3/pull/5008) +- Fix compile error when additional options follow after `crate` for `#[pyfunction]`. [#5015](https://github.com/PyO3/pyo3/pull/5015) + ## [0.24.0] - 2025-03-09 ### Packaging @@ -17,6 +102,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Add supported CPython/PyPy versions to cargo package metadata. [#4756](https://github.com/PyO3/pyo3/pull/4756) - Bump `target-lexicon` dependency to 0.13. [#4822](https://github.com/PyO3/pyo3/pull/4822) - Add optional `jiff` dependency to add conversions for `jiff` datetime types. [#4823](https://github.com/PyO3/pyo3/pull/4823) +- Add optional `uuid` dependency to add conversions for `uuid::Uuid`. [#4864](https://github.com/PyO3/pyo3/pull/4864) - Bump minimum supported `inventory` version to 0.3.5. [#4954](https://github.com/PyO3/pyo3/pull/4954) ### Added @@ -25,7 +111,6 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Add `PyCallArgs` trait for passing arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance. [#4768](https://github.com/PyO3/pyo3/pull/4768) - Add `#[pyo3(default = ...']` option for `#[derive(FromPyObject)]` to set a default value for extracted fields of named structs. [#4829](https://github.com/PyO3/pyo3/pull/4829) - Add `#[pyo3(into_py_with = ...)]` option for `#[derive(IntoPyObject, IntoPyObjectRef)]`. [#4850](https://github.com/PyO3/pyo3/pull/4850) -- Add uuid to/from python conversions. [#4864](https://github.com/PyO3/pyo3/pull/4864) - Add FFI definitions `PyThreadState_GetFrame` and `PyFrame_GetBack`. [#4866](https://github.com/PyO3/pyo3/pull/4866) - Optimize `last` for `BoundListIterator`, `BoundTupleIterator` and `BorrowedTupleIterator`. [#4878](https://github.com/PyO3/pyo3/pull/4878) - Optimize `Iterator::count()` for `PyDict`, `PyList`, `PyTuple` & `PySet`. [#4878](https://github.com/PyO3/pyo3/pull/4878) @@ -60,6 +145,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h ## [0.23.5] - 2025-02-22 + ### Packaging - Add support for PyPy3.11 [#4760](https://github.com/PyO3/pyo3/pull/4760) @@ -109,7 +195,6 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Fix unresolved symbol link failures on Windows when compiling for Python 3.13t using the `generate-import-lib` feature. [#4749](https://github.com/PyO3/pyo3/pull/4749) - Fix compile-time regression in PyO3 0.23.0 where changing `PYO3_CONFIG_FILE` would not reconfigure PyO3 for the new interpreter. [#4758](https://github.com/PyO3/pyo3/pull/4758) - ## [0.23.2] - 2024-11-25 ### Added @@ -2113,7 +2198,10 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.24.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.25.0...HEAD +[0.25.0]: https://github.com/pyo3/pyo3/compare/v0.24.2...v0.25.0 +[0.24.2]: https://github.com/pyo3/pyo3/compare/v0.24.1...v0.24.2 +[0.24.1]: https://github.com/pyo3/pyo3/compare/v0.24.0...v0.24.1 [0.24.0]: https://github.com/pyo3/pyo3/compare/v0.23.5...v0.24.0 [0.23.5]: https://github.com/pyo3/pyo3/compare/v0.23.4...v0.23.5 [0.23.4]: https://github.com/pyo3/pyo3/compare/v0.23.3...v0.23.4 diff --git a/Cargo.toml b/Cargo.toml index f0047d1a162..55e4a29170c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.25.0-dev" +version = "0.25.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -15,16 +15,15 @@ edition = "2021" rust-version = "1.63" [dependencies] -cfg-if = "1.0" libc = "0.2.62" memoffset = "0.9" once_cell = "1.13" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.25.0-dev" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.25.0" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.25.0-dev", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.25.0", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -44,10 +43,14 @@ jiff-02 = { package = "jiff", version = "0.2", optional = true } num-bigint = { version = "0.4.2", optional = true } num-complex = { version = ">= 0.4.6, < 0.5", optional = true } num-rational = { version = "0.4.1", optional = true } +ordered-float = { version = "5.0.0", default-features = false, optional = true } rust_decimal = { version = "1.15", default-features = false, optional = true } +time = { version = "0.3.38", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } uuid = { version = "1.11.0", optional = true } +lock_api = { version = "0.4", optional = true } +parking_lot = { version = "0.12", optional = true} [target.'cfg(not(target_has_atomic = "64"))'.dependencies] portable-atomic = "1.0" @@ -67,9 +70,10 @@ futures = "0.3.28" tempfile = "3.12.0" static_assertions = "1.1.0" uuid = { version = "1.10.0", features = ["v4"] } +parking_lot = { version = "0.12.3", features = ["arc_lock"]} [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.25.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } [features] default = ["macros"] @@ -102,7 +106,8 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311", "pyo3-ffi/abi3-py311"] abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] -abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314", "pyo3-ffi/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-ffi/generate-import-lib"] @@ -113,6 +118,9 @@ auto-initialize = [] # Enables `Clone`ing references to Python objects `Py` which panics if the GIL is not held. py-clone = [] +parking_lot = ["dep:parking_lot", "lock_api"] +arc_lock = ["lock_api", "lock_api/arc_lock", "parking_lot?/arc_lock"] + # Optimizes PyObject to Vec conversion and so on. nightly = [] @@ -122,6 +130,7 @@ full = [ "macros", # "multiple-pymethods", # Not supported by wasm "anyhow", + "arc_lock", "bigdecimal", "chrono", "chrono-tz", @@ -131,9 +140,12 @@ full = [ "eyre", "hashbrown", "indexmap", + "lock_api", "num-bigint", "num-complex", "num-rational", + "ordered-float", + "parking_lot", "py-clone", "rust_decimal", "serde", diff --git a/Contributing.md b/Contributing.md index 054ef42fb88..40fcb1af030 100644 --- a/Contributing.md +++ b/Contributing.md @@ -179,6 +179,20 @@ Below are guidelines on what compatibility all PRs are expected to deliver for e PyO3 supports all officially supported Python versions, as well as the latest PyPy3 release. All of these versions are tested in CI. +#### Adding support for new CPython versions + +If you plan to add support for a pre-release version of CPython, here's a (non-exhaustive) checklist: + + - [ ] Wait until the last alpha release (usually alpha7), since ABI is not guranteed until the first beta release + - [ ] Add prelease_ver-dev (e.g. `3.14-dev`) to `‎.github/workflows/ci.yml`, and bump version in `noxfile.py`, `pyo3-ffi/Cargo.toml` under `max-version` within `[package.metadata.cpython]`, and `max` within `pyo3-ffi/build.rs` +- [ ] Add a new abi3-prerelease feature for the version (e.g. `abi3-py314`) + - In `pyo3-build-config/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease"] and abi3-prerelease to ["abi3"] + - In `pyo3-ffi/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-build-config/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-build-config/abi3-prerelease"] + - In `Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-ffi/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-ffi/abi3-prerelease"] + - [ ] Use `#[cfg(Py_prerelease])` (e.g. `#[cfg(Py_3_14)]`) and `#[cfg(not(Py_prerelease]))` to indicate changes between the stable branches of CPython and the pre-release + - [ ] Do not add a Rust binding to any function, struct, or global variable prefixed with `_` in CPython's headers + - [ ] Ping @ngoldbaum and @davidhewitt for assistance + ### Rust PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible. diff --git a/README.md b/README.md index 17a20382b74..47a74ec84a3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Requires Rust 1.63 or greater. PyO3 supports the following Python distributions: - CPython 3.7 or greater - PyPy 7.3 (Python 3.9+) - - GraalPy 24.0 or greater (Python 3.10+) + - GraalPy 24.2 or greater (Python 3.11+) You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. @@ -71,7 +71,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.24.0", features = ["extension-module"] } +pyo3 = { version = "0.25.0", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -140,7 +140,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.24.0" +version = "0.25.0" features = ["auto-initialize"] ``` diff --git a/Releasing.md b/Releasing.md index 545783c598c..d3a1b4cf8c4 100644 --- a/Releasing.md +++ b/Releasing.md @@ -44,9 +44,10 @@ Wait a couple of days in case anyone wants to hold up the release to add bugfixe ## 4. Put live To put live: -- 1. run `nox -s publish` to put live on crates.io -- 2. publish the release on Github -- 3. merge the release PR +- 1. merge the release PR +- 2. publish a release on GitHub targeting the release branch + +CI will automatically push to `crates.io`. ## 5. Tidy the main branch diff --git a/build.rs b/build.rs index 68a658bf285..fd28b03b79d 100644 --- a/build.rs +++ b/build.rs @@ -38,7 +38,7 @@ fn configure_pyo3() -> Result<()> { ensure_auto_initialize_ok(interpreter_config)?; for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } // Emit cfgs like `invalid_from_utf8_lint` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index 4c5471c9945..69915ff8256 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -46,7 +46,7 @@ impl PyCounter { let new_count = self.count.fetch_add(1, Ordering::Relaxed); let name = self.wraps.getattr(py, "__name__")?; - println!("{} has been called {} time(s).", name, new_count); + println!("{name} has been called {new_count} time(s)."); // After doing something, we finally forward the call to the wrapped function let ret = self.wraps.call(py, args, kwargs)?; diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index ffd73d3a0fa..2a21f6b2e53 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index fd6e6775627..92eeabb3ea8 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/noxfile.py b/examples/setuptools-rust-starter/noxfile.py index 222be921062..d3579912ea7 100644 --- a/examples/setuptools-rust-starter/noxfile.py +++ b/examples/setuptools-rust-starter/noxfile.py @@ -1,8 +1,11 @@ import nox +import sys @nox.session def python(session: nox.Session): + if sys.version_info < (3, 9): + session.skip("Python 3.9 or later is required for setuptools-rust 1.11") session.env["SETUPTOOLS_RUST_CARGO_PROFILE"] = "dev" session.install(".[dev]") session.run("pytest") diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index 7ebca2ec821..27b15118dd4 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -10,8 +10,10 @@ | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | | `freelist = N` | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | | `frozen` | Declares that your pyclass is immutable. It removes the borrow checker overhead when retrieving a shared reference to the Rust struct, but disables the ability to get a mutable reference. | +| `generic` | Implements runtime parametrization for the class following [PEP 560](https://peps.python.org/pep-0560/). | | `get_all` | Generates getters for all fields of the pyclass. | | `hash` | Implements `__hash__` using the `Hash` implementation of the underlying Rust datatype. | +| `immutable_type` | Makes the type object immutable. Supported on 3.14+ with the `abi3` feature active, or 3.10+ otherwise. | | `mapping` | Inform PyO3 that this class is a [`Mapping`][params-mapping], and so leave its implementation of sequence C-API slots empty. | | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | diff --git a/guide/src/async-await.md b/guide/src/async-await.md index f9fc33fe3c5..1e0772b8a09 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -66,8 +66,8 @@ where fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let waker = cx.waker(); - Python::with_gil(|gil| { - gil.allow_threads(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) + Python::with_gil(|py| { + py.allow_threads(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) }) } } diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 3b9f1aafa40..a3a136fe015 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -77,8 +77,8 @@ To automatically generate the `__str__` implementation using a `Display` trait i # use pyo3::prelude::*; # # #[allow(dead_code)] -# #[pyclass(str)] -# struct Coordinate { +#[pyclass(str)] +struct Coordinate { x: i32, y: i32, z: i32, @@ -104,8 +104,8 @@ For convenience, a shorthand format string can be passed to `str` as `str="
) -> Result<(), PyTraverseError> { - if let Some(obj) = &self.obj { - visit.call(obj)? - } + visit.call(&self.obj)?; Ok(()) } diff --git a/guide/src/class/thread-safety.md b/guide/src/class/thread-safety.md index 75c0b1a7423..d0cf9e10a7f 100644 --- a/guide/src/class/thread-safety.md +++ b/guide/src/class/thread-safety.md @@ -101,7 +101,7 @@ impl MyClass { } ``` -If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. +If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. Additionally, support for the `parking_lot` and `lock_api` synchronization libraries is gated behind the `parking_lot` and `lock_api` features. You can also enable the `arc_lock` feature if you need the `arc_lock` features of either library. ### Wrapping unsynchronized data diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 34299d830d1..23d6942b177 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -17,7 +17,7 @@ The table below contains the Python type and the corresponding function argument | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `PyBytes` | | `bool` | `bool` | `PyBool` | | `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyInt` | -| `float` | `f32`, `f64` | `PyFloat` | +| `float` | `f32`, `f64`, `ordered_float::NotNan`[^10], `ordered_float::OrderedFloat`[^10] | `PyFloat` | | `complex` | `num_complex::Complex`[^2] | `PyComplex` | | `fractions.Fraction`| `num_rational::Ratio`[^8] | - | | `list[T]` | `Vec` | `PyList` | @@ -37,8 +37,8 @@ The table below contains the Python type and the corresponding function argument | `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `PyDelta` | | `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | | `decimal.Decimal` | `bigdecimal::BigDecimal`[^9] | - | -| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | -| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | +| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::Ipv4Addr` | - | +| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::Ipv6Addr` | - | | `os.PathLike ` | `PathBuf`, `Path` | `PyString` | | `pathlib.Path` | `PathBuf`, `Path` | `PyString` | | `typing.Optional[T]` | `Option` | - | @@ -119,3 +119,5 @@ Finally, the following Rust types are also able to convert to Python as return v [^8]: Requires the `num-rational` optional feature. [^9]: Requires the `bigdecimal` optional feature. + +[^10]: Requires the `ordered-float` optional feature. diff --git a/guide/src/features.md b/guide/src/features.md index 4ecd14e23fa..450c1b4ea04 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -121,6 +121,10 @@ These features enable conversions between Python types and types from other Rust Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. +### `arc_lock` + +Enables Pyo3's `MutexExt` trait for all Mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) and are wrapped in an [`Arc`](https://doc.rust-lang.org/std/sync/struct.Arc.html) type. Like [`Arc`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html#method.lock_arc) + ### `bigdecimal` Adds a dependency on [bigdecimal](https://docs.rs/bigdecimal) and enables conversions into its [`BigDecimal`](https://docs.rs/bigdecimal/latest/bigdecimal/struct.BigDecimal.html) type. @@ -168,6 +172,10 @@ Adds a dependency on [jiff@0.2](https://docs.rs/jiff/0.2) and requires MSRV 1.70 - [Zoned](https://docs.rs/jiff/0.2/jiff/struct.Zoned.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) - [Timestamp](https://docs.rs/jiff/0.2/jiff/struct.Timestamp.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +### `lock_api` + +Adds a dependency on [lock_api](https://docs.rs/lock_api) and enables Pyo3's `MutexExt` trait for all mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) (like `parking_lot` or `spin`). + ### `num-bigint` Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types. @@ -180,10 +188,31 @@ Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conv Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type. +### `ordered-float` + +Adds a dependency on [ordered-float](https://docs.rs/ordered-float) and enables conversions between [ordered-float](https://docs.rs/ordered-float)'s types and Python: +- [NotNan](https://docs.rs/ordered-float/latest/ordered_float/struct.NotNan.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) +- [OrderedFloat](https://docs.rs/ordered-float/latest/ordered_float/struct.OrderedFloat.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) + +### `parking-lot` + +Adds a dependency on [parking_lot](https://docs.rs/parking_lot) and enables Pyo3's `OnceExt` & `MutexExt` traits for [`parking_lot::Once`](https://docs.rs/parking_lot/latest/parking_lot/struct.Once.html) and [`parking_lot::Mutex`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html) types. + ### `rust_decimal` Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type. +### `time` + +Adds a dependency on [time](https://docs.rs/time) and requires MSRV 1.67.1. Enables conversions between [time](https://docs.rs/time)'s types and Python: +- [Date](https://docs.rs/time/0.3.38/time/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) +- [Time](https://docs.rs/time/0.3.38/time/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) +- [OffsetDateTime](https://docs.rs/time/0.3.38/time/struct.OffsetDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [PrimitiveDateTime](https://docs.rs/time/0.3.38/time/struct.PrimitiveDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Duration](https://docs.rs/time/0.3.38/time/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [UtcOffset](https://docs.rs/time/0.3.38/time/struct.UtcOffset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [UtcDateTime](https://docs.rs/time/0.3.38/time/struct.UtcDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) + ### `serde` Enables (de)serialization of `Py` objects via [serde](https://serde.rs/). diff --git a/guide/src/migration.md b/guide/src/migration.md index 13ca506662c..3beaa5bc07c 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -3,9 +3,26 @@ This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see the [CHANGELOG](changelog.md). -## from 0.22.* to 0.23 +## from 0.24.* to 0.25 +### `AsPyPointer` removal
Click to expand +The `AsPyPointer` trait is mostly a leftover from the now removed gil-refs API. The last remaining uses were the GC API, namely `PyVisit::call`, and identity comparison (`PyAnyMethods::is` and `Py::is`). + +`PyVisit::call` has been updated to take `T: Into>>`, which allows for arguments of type `&Py`, `&Option>` and `Option<&Py>`. It is unlikely any changes are needed here to migrate. + +`PyAnyMethods::is`/`Py::is` has been updated to take `T: AsRef>>`. Additionally `AsRef>>` implementations were added for `Py`, `Bound` and `Borrowed`. Because of the existing `AsRef> for Bound` implementation this may cause inference issues in non-generic code. This can be easily migrated by switching to `as_any` instead of `as_ref` for these calls. +
+ +## from 0.23.* to 0.24 +
+Click to expand +There were no significant changes from 0.23 to 0.24 which required documenting in this guide. +
+ +## from 0.22.* to 0.23 +
+Click to expand PyO3 0.23 is a significant rework of PyO3's internals for two major improvements: - Support of Python 3.13's new freethreaded build (aka "3.13t") @@ -20,7 +37,7 @@ The sections below discuss the rationale and details of each change in more dept
### Free-threaded Python Support -
+
Click to expand PyO3 0.23 introduces initial support for the new free-threaded build of @@ -43,7 +60,7 @@ See [the guide section on free-threaded Python](free-threading.md) for more deta
### New `IntoPyObject` trait unifies to-Python conversions -
+
Click to expand PyO3 0.23 introduces a new `IntoPyObject` trait to convert Rust types into Python objects which replaces both `IntoPy` and `ToPyObject`. @@ -132,7 +149,7 @@ impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper {
### To-Python conversions changed for byte collections (`Vec`, `[u8; N]` and `SmallVec<[u8; N]>`). -
+
Click to expand With the introduction of the `IntoPyObject` trait, PyO3's macros now prefer `IntoPyObject` implementations over `IntoPy` when producing Python values. This applies to `#[pyfunction]` and `#[pymethods]` return values and also fields accessed via `#[pyo3(get)]`. @@ -170,7 +187,7 @@ This is purely additional and should just extend the possible return types.
### `gil-refs` feature removed -
+
Click to expand PyO3 0.23 completes the removal of the "GIL Refs" API in favour of the new "Bound" API introduced in PyO3 0.21. diff --git a/guide/src/module.md b/guide/src/module.md index deb516d9c97..fa756e85920 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -124,6 +124,9 @@ mod my_extension { #[pymodule_export] use super::double; // Exports the double function as part of the module + #[pymodule_export] + const PI: f64 = std::f64::consts::PI; // Exports PI constant as part of the module + #[pyfunction] // This will be part of the module fn triple(x: usize) -> usize { x * 3 diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md index 2e5cf3c589e..b64c062eec7 100644 --- a/guide/src/python-from-rust/function-calls.md +++ b/guide/src/python-from-rust/function-calls.md @@ -59,13 +59,13 @@ fn main() -> PyResult<()> { ## Creating keyword arguments -For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. +For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. To pass keyword arguments of different types, construct a `PyDict` object. ```rust use pyo3::prelude::*; -use pyo3::types::IntoPyDict; +use pyo3::types::{PyDict, IntoPyDict}; use std::collections::HashMap; -use pyo3_ffi::c_str; +use pyo3::ffi::c_str; fn main() -> PyResult<()> { let key1 = "key1"; @@ -102,7 +102,13 @@ fn main() -> PyResult<()> { kwargs.insert(key1, 1); fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; + // pass arguments of different types as PyDict + let kwargs = PyDict::new(py); + kwargs.set_item(key1, val1)?; + kwargs.set_item(key2, "string")?; + fun.call(py, (), Some(&kwargs))?; + Ok(()) }) } -``` \ No newline at end of file +``` diff --git a/guide/src/python-typing-hints.md b/guide/src/python-typing-hints.md index 43a15a21a6e..eb39bb35fb0 100644 --- a/guide/src/python-typing-hints.md +++ b/guide/src/python-typing-hints.md @@ -41,7 +41,7 @@ As we can see, those are not full definitions containing implementation, but jus ### What do the PEPs say? -At the time of writing this documentation, the `pyi` files are referenced in three PEPs. +At the time of writing this documentation, the `pyi` files are referenced in four PEPs. [PEP8 - Style Guide for Python Code - #Function Annotations](https://www.python.org/dev/peps/pep-0008/#function-annotations) (last point) recommends all third party library creators to provide stub files as the source of knowledge about the package for type checker tools. @@ -55,6 +55,8 @@ It contains a specification for them (highly recommended reading, since it conta [PEP561 - Distributing and Packaging Type Information](https://www.python.org/dev/peps/pep-0561/) describes in detail how to build packages that will enable type checking. In particular it contains information about how the stub files must be distributed in order for type checkers to use them. +[PEP560 - Core support for typing module and generic types](https://www.python.org/dev/peps/pep-0560/) describes the details on how Python's type system internally supports generics, including both runtime behavior and integration with static type checkers. + ## How to do it? [PEP561](https://www.python.org/dev/peps/pep-0561/) recognizes three ways of distributing type information: @@ -165,3 +167,77 @@ class Car: :return: the name of the color our great algorithm thinks is the best for this car """ ``` + +### Supporting Generics + +Type annotations can also be made generic in Python. They are useful for working +with different types while maintaining type safety. Usually, generic classes +inherit from the `typing.Generic` metaclass. + +Take for example the following `.pyi` file that specifies a `Car` that can +accept multiple types of wheels: + +```python +from typing import Generic, TypeVar + +W = TypeVar('W') + +class Car(Generic[W]): + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... + + def change_wheel(self, wheel_number: int, wheel: W) -> None: ... +``` + +This way, the end-user can specify the type with variables such as `truck: Car[SteelWheel] = ...` +and `f1_car: Car[AlloyWheel] = ...`. + +There is also a special syntax for specifying generic types in Python 3.12+: + +```python +class Car[W]: + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... +``` + +#### Runtime Behaviour + +Stub files (`pyi`) are only useful for static type checkers and ignored at runtime. Therefore, +PyO3 classes do not inherit from `typing.Generic` even if specified in the stub files. + +This can cause some runtime issues, as annotating a variable like `f1_car: Car[AlloyWheel] = ...` +can make Python call magic methods that are not defined. + +To overcome this limitation, implementers can pass the `generic` parameter to `pyclass` in Rust: + +```rust ignore +#[pyclass(generic)] +``` + +#### Advanced Users + +`#[pyclass(generic)]` implements a very simple runtime behavior that accepts +any generic argument. Advanced users can opt to manually implement +[`__class_geitem__`](https://docs.python.org/3/reference/datamodel.html#emulating-generic-types) +for the generic class to have more control. + +```rust ignore +impl MyClass { + #[classmethod] + #[pyo3(signature = (key, /))] + pub fn __class_getitem__( + cls: &Bound<'_, PyType>, + key: &Bound<'_, PyAny>, + ) -> PyResult { + /* implementation details */ + } +} +``` + +Note that [`pyo3::types::PyGenericAlias`][pygenericalias] can be helfpul when implementing +`__class_geitem__` as it can create [`types.GenericAlias`][genericalias] objects from Rust. + +[pygenericalias]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.pygenericalias +[genericalias]: https://docs.python.org/3/library/types.html#types.GenericAlias \ No newline at end of file diff --git a/newsfragments/3977.added.md b/newsfragments/3977.added.md deleted file mode 100644 index 58f116cd837..00000000000 --- a/newsfragments/3977.added.md +++ /dev/null @@ -1 +0,0 @@ -Basic introspection and stub generation based on metadata embedded in produced cdylib. \ No newline at end of file diff --git a/newsfragments/4969.added.md b/newsfragments/4969.added.md deleted file mode 100644 index 7a59e9aef6f..00000000000 --- a/newsfragments/4969.added.md +++ /dev/null @@ -1 +0,0 @@ -Added `abi3-py313` feature \ No newline at end of file diff --git a/newsfragments/4970.changed.md b/newsfragments/4970.changed.md deleted file mode 100644 index 7ea1b6ae0cd..00000000000 --- a/newsfragments/4970.changed.md +++ /dev/null @@ -1 +0,0 @@ -- Make `datetime` rust wrappers compatible with `abi3` feature diff --git a/newsfragments/4978.added.md b/newsfragments/4978.added.md deleted file mode 100644 index 5518cee89d8..00000000000 --- a/newsfragments/4978.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement getattr_opt in `PyAnyMethods` \ No newline at end of file diff --git a/newsfragments/4981.fixed.md b/newsfragments/4981.fixed.md deleted file mode 100644 index 0ac31b19a11..00000000000 --- a/newsfragments/4981.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. diff --git a/newsfragments/4982.removed.md b/newsfragments/4982.removed.md deleted file mode 100644 index 61f9dedbc48..00000000000 --- a/newsfragments/4982.removed.md +++ /dev/null @@ -1 +0,0 @@ -Remove all functionality deprecated in PyO3 0.23. diff --git a/newsfragments/4984.added.md b/newsfragments/4984.added.md deleted file mode 100644 index 63559617812..00000000000 --- a/newsfragments/4984.added.md +++ /dev/null @@ -1 +0,0 @@ -Added PyInt constructor for all supported number types (i32, u32, i64, u64, isize, usize) \ No newline at end of file diff --git a/newsfragments/4988.fixed.md b/newsfragments/4988.fixed.md deleted file mode 100644 index 1a050b905e3..00000000000 --- a/newsfragments/4988.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `Probe` class naming issue with `#[pymethods]` diff --git a/newsfragments/4992.added.md b/newsfragments/4992.added.md deleted file mode 100644 index 36cb9a4e08e..00000000000 --- a/newsfragments/4992.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `pyo3::sync::with_critical_section2` binding diff --git a/newsfragments/5002.fixed.md b/newsfragments/5002.fixed.md deleted file mode 100644 index ad9b0091a79..00000000000 --- a/newsfragments/5002.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). diff --git a/newsfragments/5008.fixed.md b/newsfragments/5008.fixed.md deleted file mode 100644 index 6b431d6e437..00000000000 --- a/newsfragments/5008.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `PyString::from_object`, avoid out of bounds reads by null terminating the `encoding` and `errors` parameters \ No newline at end of file diff --git a/newsfragments/5010.removed.md b/newsfragments/5010.removed.md deleted file mode 100644 index 9b42e682154..00000000000 --- a/newsfragments/5010.removed.md +++ /dev/null @@ -1 +0,0 @@ -Remove deprecated `IntoPy` and `ToPyObject` traits \ No newline at end of file diff --git a/newsfragments/5011.added.md b/newsfragments/5011.added.md deleted file mode 100644 index 741a25b3707..00000000000 --- a/newsfragments/5011.added.md +++ /dev/null @@ -1 +0,0 @@ -Added conversion support for `bigdecimal::BigDecimal` to and from python `Decimal` type \ No newline at end of file diff --git a/newsfragments/5013.added.md b/newsfragments/5013.added.md deleted file mode 100644 index 0becc3ee1e9..00000000000 --- a/newsfragments/5013.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. diff --git a/newsfragments/5015.fixed.md b/newsfragments/5015.fixed.md deleted file mode 100644 index b1da048828f..00000000000 --- a/newsfragments/5015.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixes compile error if more options followed after `crate` for `#[pyfunction]`. \ No newline at end of file diff --git a/newsfragments/5030.fixed.md b/newsfragments/5030.fixed.md deleted file mode 100644 index 4cd23405a6c..00000000000 --- a/newsfragments/5030.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixes accidentally emitting `unused_imports` lint in `macro_rules` context. \ No newline at end of file diff --git a/newsfragments/5037.fixed.md b/newsfragments/5037.fixed.md deleted file mode 100644 index 770a136f4f1..00000000000 --- a/newsfragments/5037.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Ensure we correctly pick up the shared state for conda-based Python installation when reading information from sysconfigdata. diff --git a/newsfragments/5048.fixed.md b/newsfragments/5048.fixed.md deleted file mode 100644 index de3f79812b1..00000000000 --- a/newsfragments/5048.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix size of `PyCodeObject::_co_instrumentation_version` ffi struct member on Python 3.13 for systems where `uintptr_t` is not 64 bits. diff --git a/newsfragments/5050.fixed.md b/newsfragments/5050.fixed.md deleted file mode 100644 index 576809467c6..00000000000 --- a/newsfragments/5050.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix struct-type complex enum variant fields exposing raw identifiers as `r#ident` in Python bindings. diff --git a/newsfragments/5054.added.md b/newsfragments/5054.added.md deleted file mode 100644 index 1400624a577..00000000000 --- a/newsfragments/5054.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `From`/`Into` for `Borrowed` -> `Py` diff --git a/newsfragments/5121.added.md b/newsfragments/5121.added.md new file mode 100644 index 00000000000..fd79e5a8ea7 --- /dev/null +++ b/newsfragments/5121.added.md @@ -0,0 +1 @@ +Add `PyBytes_AS_STRING` diff --git a/newsfragments/5121.changed.md b/newsfragments/5121.changed.md new file mode 100644 index 00000000000..49b317cb96b --- /dev/null +++ b/newsfragments/5121.changed.md @@ -0,0 +1,2 @@ +Enable vectorcall methods on GraalPy +Call Py_Is function on GraalPy diff --git a/noxfile.py b/noxfile.py index f17d40aff14..94e0f392044 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,6 @@ List, Optional, Tuple, - Generator, ) @@ -72,6 +71,9 @@ def _supported_interpreter_versions( min_minor = int(min_version.split(".")[1]) max_minor = int(max_version.split(".")[1]) versions = [f"{major}.{minor}" for minor in range(min_minor, max_minor + 1)] + # Add free-threaded builds for 3.13+ + if python_impl == "cpython": + versions += [f"{major}.{minor}t" for minor in range(13, max_minor + 1)] return versions @@ -90,17 +92,49 @@ def test_rust(session: nox.Session): _run_cargo_test(session, package="pyo3-build-config") _run_cargo_test(session, package="pyo3-macros-backend") _run_cargo_test(session, package="pyo3-macros") - _run_cargo_test(session, package="pyo3-ffi") - _run_cargo_test(session) - # the free-threaded build ignores abi3, so we skip abi3 - # tests to avoid unnecessarily running the tests twice - if not FREE_THREADED_BUILD: - _run_cargo_test(session, features="abi3") - if "skip-full" not in session.posargs: - _run_cargo_test(session, features="full jiff-02") - if not FREE_THREADED_BUILD: - _run_cargo_test(session, features="abi3 full jiff-02") + extra_flags = [] + # pypy and graalpy don't have Py_Initialize APIs, so we can only + # build the main tests, not run them + if sys.implementation.name in ("pypy", "graalpy"): + extra_flags.append("--no-run") + + _run_cargo_test(session, package="pyo3-ffi", extra_flags=extra_flags) + + extra_flags.append("--no-default-features") + + for feature_set in _get_feature_sets(): + flags = extra_flags.copy() + print(feature_set) + + if feature_set is None or "full" not in feature_set: + # doctests require at least the macros feature, which is + # activated by the full feature set + # + # using `--all-targets` makes cargo run everything except doctests + flags.append("--all-targets") + + # We need to pass the feature set to the test command + # so that it can be used in the test code + # (e.g. for `#[cfg(feature = "abi3-py37")]`) + if feature_set and "abi3" in feature_set and FREE_THREADED_BUILD: + # free-threaded builds don't support abi3 yet + continue + + _run_cargo_test(session, features=feature_set, extra_flags=flags) + + if ( + feature_set + and "abi3" in feature_set + and "full" in feature_set + and sys.version_info >= (3, 7) + ): + # run abi3-py37 tests to check abi3 forward compatibility + _run_cargo_test( + session, + features=feature_set.replace("abi3", "abi3-py37"), + extra_flags=flags, + ) @nox.session(name="test-py", venv_backend="none") @@ -179,7 +213,8 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: _run_cargo( session, "clippy", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", "--", @@ -256,7 +291,8 @@ def _check(env: Dict[str, str]) -> None: _run_cargo( session, "check", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", env=env, @@ -426,6 +462,10 @@ def docs(session: nox.Session) -> None: features = "full" + if get_rust_version()[:2] >= (1, 67): + # time needs MSRC 1.67+ + features += ",time" + if get_rust_version()[:2] >= (1, 70): # jiff needs MSRC 1.70+ features += ",jiff-02" @@ -486,6 +526,7 @@ def check_guide(session: nox.Session): "--include-fragments", str(PYO3_GUIDE_SRC), *remap_args, + "--accept=200,429", *session.posargs, ) # check external links in the docs @@ -497,6 +538,7 @@ def check_guide(session: nox.Session): *remap_args, f"--exclude=file://{PYO3_DOCS_TARGET}", "--exclude=http://www.adobe.com/", + "--accept=200,429", *session.posargs, ) @@ -701,11 +743,11 @@ def test_version_limits(session: nox.Session): config_file.set("CPython", "3.6") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.14" not in PY_VERSIONS - config_file.set("CPython", "3.14") + assert "3.15" not in PY_VERSIONS + config_file.set("CPython", "3.15") _run_cargo(session, "check", env=env, expect_error=True) - # 3.14 CPython should build with forward compatibility + # 3.15 CPython should build with forward compatibility env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" _run_cargo(session, "check", env=env) @@ -725,7 +767,10 @@ def check_feature_powerset(session: nox.Session): cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text()) - EXPECTED_ABI3_FEATURES = {f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS} + # free-threaded builds do not support ABI3 (yet) + EXPECTED_ABI3_FEATURES = { + f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS if not ver.endswith("t") + } EXCLUDED_FROM_FULL = { "nightly", @@ -813,8 +858,8 @@ def update_ui_tests(session: nox.Session): env["TRYBUILD"] = "overwrite" command = ["test", "--test", "test_compile_error"] _run_cargo(session, *command, env=env) - _run_cargo(session, *command, "--features=full,jiff-02", env=env) - _run_cargo(session, *command, "--features=abi3,full,jiff-02", env=env) + _run_cargo(session, *command, "--features=full,jiff-02,time", env=env) + _run_cargo(session, *command, "--features=abi3,full,jiff-02,time", env=env) @nox.session(name="test-introspection") @@ -860,6 +905,13 @@ def get_rust_version() -> Tuple[int, int, int, List[str]]: return (*map(int, version_number.split(".")), extra) +def is_rust_nightly() -> bool: + for line in _get_rust_info(): + if line.startswith(_RELEASE_LINE_START): + return line.strip().endswith("-nightly") + return False + + def _get_rust_default_target() -> str: for line in _get_rust_info(): if line.startswith(_HOST_LINE_START): @@ -867,30 +919,28 @@ def _get_rust_default_target() -> str: @lru_cache() -def _get_feature_sets() -> Generator[Tuple[str, ...], None, None]: - """Returns feature sets to use for clippy job""" +def _get_feature_sets() -> Tuple[Optional[str], ...]: + """Returns feature sets to use for Rust jobs""" cargo_target = os.getenv("CARGO_BUILD_TARGET", "") - yield from ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ) - features = "full" if "wasm32-wasip1" not in cargo_target: # multiple-pymethods not supported on wasm features += ",multiple-pymethods" + if get_rust_version()[:2] >= (1, 67): + # time needs MSRC 1.67+ + features += ",time" + if get_rust_version()[:2] >= (1, 70): # jiff needs MSRC 1.70+ features += ",jiff-02" - yield (f"--features={features}",) - yield (f"--features=abi3,{features}",) + if is_rust_nightly(): + features += ",nightly" + + return (None, "abi3", features, f"abi3,{features}") _RELEASE_LINE_START = "release: " @@ -954,19 +1004,24 @@ def _run_cargo_test( package: Optional[str] = None, features: Optional[str] = None, env: Optional[Dict[str, str]] = None, + extra_flags: Optional[List[str]] = None, ) -> None: command = ["cargo"] if "careful" in session.posargs: # do explicit setup so failures in setup can be seen _run_cargo(session, "careful", "setup") command.append("careful") + command.extend(("test", "--no-fail-fast")) + if "release" in session.posargs: command.append("--release") if package: command.append(f"--package={package}") if features: command.append(f"--features={features}") + if extra_flags: + command.extend(extra_flags) _run(session, *command, external=True, env=env or {}) @@ -1015,6 +1070,11 @@ def set( self, implementation: str, version: str, build_flags: Iterable[str] = () ) -> None: """Set the contents of this config file to the given implementation and version.""" + if version.endswith("t"): + # Free threaded versions pass the support in config file through a flag + version = version[:-1] + build_flags = (*build_flags, "Py_GIL_DISABLED") + self._config_file.seek(0) self._config_file.truncate(0) self._config_file.write( diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 86456e33cb5..8070860a1e9 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.25.0-dev" +version = "0.25.0" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,8 @@ abi3-py39 = ["abi3-py310"] abi3-py310 = ["abi3-py311"] abi3-py311 = ["abi3-py312"] abi3-py312 = ["abi3-py313"] -abi3-py313 = ["abi3"] +abi3-py313 = ["abi3-py314"] +abi3-py314 = ["abi3"] [package.metadata.docs.rs] features = ["resolve-config"] diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 87c59a998b4..b11d02fd581 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -68,7 +68,7 @@ impl std::fmt::Display for ErrorReport<'_> { writeln!(f, "\ncaused by:")?; let mut index = 0; while let Some(some_source) = source { - writeln!(f, " - {}: {}", index, some_source)?; + writeln!(f, " - {index}: {some_source}")?; source = some_source.source(); index += 1; } diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 96eb67657ea..9485b8a0394 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -58,7 +58,7 @@ pub fn cargo_env_var(var: &str) -> Option { /// the variable changes. pub fn env_var(var: &str) -> Option { if cfg!(feature = "resolve-config") { - println!("cargo:rerun-if-env-changed={}", var); + println!("cargo:rerun-if-env-changed={var}"); } #[cfg(test)] { @@ -180,7 +180,7 @@ impl InterpreterConfig { let mut out = vec![]; for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { - out.push(format!("cargo:rustc-cfg=Py_3_{}", i)); + out.push(format!("cargo:rustc-cfg=Py_3_{i}")); } match self.implementation { @@ -199,7 +199,7 @@ impl InterpreterConfig { BuildFlag::Py_GIL_DISABLED => { out.push("cargo:rustc-cfg=Py_GIL_DISABLED".to_owned()) } - flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)), + flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{flag}\"")), } } @@ -338,7 +338,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let lib_dir = if cfg!(windows) { map.get("base_prefix") - .map(|base_prefix| format!("{}\\libs", base_prefix)) + .map(|base_prefix| format!("{base_prefix}\\libs")) } else { map.get("libdir").cloned() }; @@ -666,7 +666,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { - writeln!(writer, "extra_build_script_line={}", line) + writeln!(writer, "extra_build_script_line={line}") .context("failed to write extra_build_script_line")?; } Ok(()) @@ -853,7 +853,7 @@ fn is_abi3() -> bool { /// Must be called from a PyO3 crate build script. pub fn get_abi3_version() -> Option { let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()); + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{i}")).is_some()); minor_version.map(|minor| PythonVersion { major: 3, minor }) } @@ -1121,8 +1121,8 @@ pub enum BuildFlag { impl Display for BuildFlag { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - BuildFlag::Other(flag) => write!(f, "{}", flag), - _ => write!(f, "{:?}", self), + BuildFlag::Other(flag) => write!(f, "{flag}"), + _ => write!(f, "{self:?}"), } } } @@ -1202,7 +1202,7 @@ impl BuildFlags { for k in &BuildFlags::ALL { use std::fmt::Write; - writeln!(&mut script, "print(config.get('{}', '0'))", k).unwrap(); + writeln!(&mut script, "print(config.get('{k}', '0'))").unwrap(); } let stdout = run_python_script(interpreter.as_ref(), &script)?; @@ -1240,7 +1240,7 @@ impl Display for BuildFlags { } else { write!(f, ",")?; } - write!(f, "{}", flag)?; + write!(f, "{flag}")?; } Ok(()) } @@ -1427,7 +1427,7 @@ pub fn find_all_sysconfigdata(cross: &CrossCompileConfig) -> Result fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { let pypy_version_pat = if let Some(v) = v { - format!("pypy{}", v) + format!("pypy{v}") } else { "pypy3.".into() }; @@ -1436,7 +1436,7 @@ fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { let graalpy_version_pat = if let Some(v) = v { - format!("graalpy{}", v) + format!("graalpy{v}") } else { "graalpy2".into() }; @@ -1445,7 +1445,7 @@ fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { let cpython_version_pat = if let Some(v) = v { - format!("python{}", v) + format!("python{v}") } else { "python3.".into() }; @@ -1759,7 +1759,7 @@ fn default_lib_name_unix( ) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => Ok(format!("python{}", ld_version)), + Some(ld_version) => Ok(format!("python{ld_version}")), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone @@ -1776,7 +1776,7 @@ fn default_lib_name_unix( } }, PythonImplementation::PyPy => match ld_version { - Some(ld_version) => Ok(format!("pypy{}-c", ld_version)), + Some(ld_version) => Ok(format!("pypy{ld_version}-c")), None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), }, diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 68d597a8f2e..f47c16f425d 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -46,7 +46,7 @@ use target_lexicon::OperatingSystem; pub fn use_pyo3_cfgs() { print_expected_cfgs(); for cargo_command in get().build_script_outputs() { - println!("{}", cargo_command) + println!("{cargo_command}") } } @@ -102,12 +102,7 @@ fn _add_python_framework_link_args( ) { if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { - writeln!( - writer, - "cargo:rustc-link-arg=-Wl,-rpath,{}", - framework_prefix - ) - .unwrap(); + writeln!(writer, "cargo:rustc-link-arg=-Wl,-rpath,{framework_prefix}").unwrap(); } } } @@ -173,12 +168,12 @@ fn print_feature_cfg(minor_version_required: u32, cfg: &str) { let minor_version = rustc_minor_version().unwrap_or(0); if minor_version >= minor_version_required { - println!("cargo:rustc-cfg={}", cfg); + println!("cargo:rustc-cfg={cfg}"); } // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before if minor_version >= 80 { - println!("cargo:rustc-check-cfg=cfg({})", cfg); + println!("cargo:rustc-check-cfg=cfg({cfg})"); } } @@ -221,8 +216,7 @@ pub fn print_expected_cfgs() { // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) - // FIXME: support cfg(Py_3_14) as well due to PyGILState_Ensure - for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=std::cmp::max(14, impl_::ABI3_MAX_MINOR + 1) { + for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=impl_::ABI3_MAX_MINOR + 1 { println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); } } diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index 67808888e1e..e7cfbe40df3 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -1,6 +1,24 @@ use std::env; use std::path::PathBuf; +#[derive(Debug)] +struct ParseCallbacks; + +impl bindgen::callbacks::ParseCallbacks for ParseCallbacks { + // these are anonymous fields and structs in CPython that we needed to + // invent names for. Bindgen seems to generate stable names, so we remap the + // automatically generated names to the names we invented in the FFI + fn item_name(&self, _original_item_name: &str) -> Option { + if _original_item_name == "_object__bindgen_ty_1__bindgen_ty_1" { + Some("PyObjectObFlagsAndRefcnt".into()) + } else if _original_item_name == "_object__bindgen_ty_1" { + Some("PyObjectObRefcnt".into()) + } else { + None + } + } +} + fn main() { let config = pyo3_build_config::get(); let python_include_dir = config @@ -29,6 +47,7 @@ fn main() { .header("wrapper.h") .clang_args(clang_args) .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .parse_callbacks(Box::new(ParseCallbacks)) // blocklist some values which apparently have conflicting definitions on unix .blocklist_item("FP_NORMAL") .blocklist_item("FP_SUBNORMAL") diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index c6cee5c7ec3..41092b9020e 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -9,11 +9,6 @@ const PY_3_12: PythonVersion = PythonVersion { minor: 12, }; -const PY_3_13: PythonVersion = PythonVersion { - major: 3, - minor: 13, -}; - /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -55,16 +50,6 @@ pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStrea .strip_suffix(".html") .unwrap(); - if struct_name == "_PyCoLineInstrumentationData" - && pyo3_build_config::get().version == PY_3_13 - { - // private type, fields changed name in 3.13.2 -> 3.13.3 - // - // PyO3 0.25 will remove this struct, ignoring temporarily just to unblock CI - // changed, the size stayed the same. - continue; - } - let struct_ident = Ident::new(struct_name, Span::call_site()); output.extend(quote!(#macro_name!(#struct_ident);)); } diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 8b75c978c2e..647b76fbdcd 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.25.0-dev" +version = "0.25.0" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -34,7 +34,8 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311"] abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312"] -abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-build-config/python3-dll-a"] @@ -43,14 +44,14 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } [lints] workspace = true [package.metadata.cpython] min-version = "3.7" -max-version = "3.13" # inclusive +max-version = "3.14" # inclusive [package.metadata.pypy] min-version = "3.9" diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index e817a1a73d7..09be0a06f66 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -41,13 +41,13 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3-ffi] -version = "0.24.0" +version = "0.25.0" features = ["extension-module"] [build-dependencies] # This is only necessary if you need to configure your build based on # the Python version or the compile-time configuration for the interpreter. -pyo3_build_config = "0.24.0" +pyo3_build_config = "0.25.0" ``` If you need to use conditional compilation based on Python version or how diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 096614c7961..6776cd80476 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -17,7 +17,7 @@ const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { min: PythonVersion { major: 3, minor: 7 }, max: PythonVersion { major: 3, - minor: 13, + minor: 14, }, }; @@ -176,7 +176,7 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { ); if let Some(lib_dir) = &interpreter_config.lib_dir { - println!("cargo:rustc-link-search=native={}", lib_dir); + println!("cargo:rustc-link-search=native={lib_dir}"); } Ok(()) @@ -207,12 +207,12 @@ fn configure_pyo3() -> Result<()> { } for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } // Extra lines come last, to support last write wins. for line in &interpreter_config.extra_build_script_lines { - println!("{}", line); + println!("{line}"); } // Emit cfgs like `invalid_from_utf8_lint` diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index a79ec43f271..59639d14de8 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -93,7 +93,7 @@ extern "C" { kwnames: *mut PyObject, ) -> *mut PyObject; - #[cfg(any(Py_3_12, all(Py_3_9, not(any(Py_LIMITED_API, PyPy, GraalPy)))))] + #[cfg(any(Py_3_12, all(Py_3_9, not(any(Py_LIMITED_API, PyPy)))))] pub fn PyObject_VectorcallMethod( name: *mut PyObject, args: *const *mut PyObject, diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index 24a97bcc31b..d27dfa8b0ec 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -17,7 +17,7 @@ pub struct PyByteArrayObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyByteArrayObject); +opaque_struct!(pub PyByteArrayObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/code.rs b/pyo3-ffi/src/code.rs index d28f68cded7..296b17f6aa4 100644 --- a/pyo3-ffi/src/code.rs +++ b/pyo3-ffi/src/code.rs @@ -1,4 +1,4 @@ // This header doesn't exist in CPython, but Include/cpython/code.h does. We add // this here so that PyCodeObject has a definition under the limited API. -opaque_struct!(PyCodeObject); +opaque_struct!(pub PyCodeObject); diff --git a/pyo3-ffi/src/compat/mod.rs b/pyo3-ffi/src/compat/mod.rs index 11f2912848e..044ea46762b 100644 --- a/pyo3-ffi/src/compat/mod.rs +++ b/pyo3-ffi/src/compat/mod.rs @@ -52,8 +52,10 @@ macro_rules! compat_function { mod py_3_10; mod py_3_13; +mod py_3_14; mod py_3_9; pub use self::py_3_10::*; pub use self::py_3_13::*; +pub use self::py_3_14::*; pub use self::py_3_9::*; diff --git a/pyo3-ffi/src/compat/py_3_10.rs b/pyo3-ffi/src/compat/py_3_10.rs index c6e8c2cb5ca..d962fb3bc7e 100644 --- a/pyo3-ffi/src/compat/py_3_10.rs +++ b/pyo3-ffi/src/compat/py_3_10.rs @@ -17,3 +17,29 @@ compat_function!( obj } ); + +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn PyModule_AddObjectRef( + module: *mut crate::PyObject, + name: *const std::os::raw::c_char, + value: *mut crate::PyObject, + ) -> std::os::raw::c_int { + if value.is_null() && crate::PyErr_Occurred().is_null() { + crate::PyErr_SetString( + crate::PyExc_SystemError, + c_str!("PyModule_AddObjectRef() must be called with an exception raised if value is NULL").as_ptr(), + ); + return -1; + } + + crate::Py_XINCREF(value); + let result = crate::PyModule_AddObject(module, name, value); + if result < 0 { + crate::Py_XDECREF(value); + } + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_13.rs b/pyo3-ffi/src/compat/py_3_13.rs index 59289cb76ae..96c90e7eb66 100644 --- a/pyo3-ffi/src/compat/py_3_13.rs +++ b/pyo3-ffi/src/compat/py_3_13.rs @@ -104,3 +104,18 @@ compat_function!( crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) } ); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyModule_Add( + module: *mut crate::PyObject, + name: *const std::os::raw::c_char, + value: *mut crate::PyObject, + ) -> std::os::raw::c_int { + let result = crate::compat::PyModule_AddObjectRef(module, name, value); + crate::Py_XDECREF(value); + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_14.rs b/pyo3-ffi/src/compat/py_3_14.rs new file mode 100644 index 00000000000..6fdaef17488 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_14.rs @@ -0,0 +1,26 @@ +compat_function!( + originally_defined_for(all(Py_3_14, not(Py_LIMITED_API))); + + #[inline] + pub unsafe fn Py_HashBuffer( + ptr: *const std::ffi::c_void, + len: crate::Py_ssize_t, + ) -> crate::Py_hash_t { + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + { + crate::_Py_HashBytes(ptr, len) + } + + #[cfg(any(Py_LIMITED_API, PyPy))] + { + let bytes = crate::PyBytes_FromStringAndSize(ptr as *const std::os::raw::c_char, len); + if bytes.is_null() { + -1 + } else { + let result = crate::PyObject_Hash(bytes); + crate::Py_DECREF(bytes); + result + } + } + } +); diff --git a/pyo3-ffi/src/compat/py_3_9.rs b/pyo3-ffi/src/compat/py_3_9.rs index 285f2b2ae7e..6b3521cc167 100644 --- a/pyo3-ffi/src/compat/py_3_9.rs +++ b/pyo3-ffi/src/compat/py_3_9.rs @@ -1,7 +1,6 @@ compat_function!( originally_defined_for(all( not(PyPy), - not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 )); @@ -12,7 +11,7 @@ compat_function!( ); compat_function!( - originally_defined_for(all(Py_3_9, not(any(Py_LIMITED_API, PyPy, GraalPy)))); + originally_defined_for(all(Py_3_9, not(any(Py_LIMITED_API, PyPy)))); #[inline] pub unsafe fn PyObject_CallMethodNoArgs(obj: *mut crate::PyObject, name: *mut crate::PyObject) -> *mut crate::PyObject { diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index 6ada1a754ef..b9d9dd47dc9 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,12 +1,12 @@ use crate::{PyObject, Py_ssize_t}; -#[cfg(any(all(Py_3_8, not(any(PyPy, GraalPy))), not(Py_3_11)))] +#[cfg(any(all(Py_3_8, not(PyPy)), not(Py_3_11)))] use std::os::raw::c_char; use std::os::raw::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_8, not(PyPy)))] use crate::{ vectorcallfunc, PyCallable_Check, PyThreadState, PyThreadState_GET, PyTuple_Check, PyType_HasFeature, Py_TPFLAGS_HAVE_VECTORCALL, @@ -23,7 +23,7 @@ extern "C" { const _PY_FASTCALL_SMALL_STACK: size_t = 5; extern "C" { - #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_8, not(PyPy)))] pub fn _Py_CheckFunctionResult( tstate: *mut PyThreadState, callable: *mut PyObject, @@ -31,7 +31,7 @@ extern "C" { where_: *const c_char, ) -> *mut PyObject; - #[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_8, not(PyPy)))] pub fn _PyObject_MakeTpCall( tstate: *mut PyThreadState, callable: *mut PyObject, @@ -52,7 +52,7 @@ pub unsafe fn PyVectorcall_NARGS(n: size_t) -> Py_ssize_t { n.try_into().expect("cannot fail due to mask") } -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_8, not(PyPy)))] #[inline(always)] pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option { assert!(!callable.is_null()); @@ -67,7 +67,7 @@ pub unsafe fn PyVectorcall_Function(callable: *mut PyObject) -> Option *mut PyObject { _PyObject_VectorcallTstate( @@ -166,7 +166,7 @@ extern "C" { pub fn _PyObject_CallNoArg(func: *mut PyObject) -> *mut PyObject; } -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_8, not(PyPy)))] #[inline(always)] pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *mut PyObject { assert!(!arg.is_null()); @@ -177,7 +177,7 @@ pub unsafe fn PyObject_CallOneArg(func: *mut PyObject, arg: *mut PyObject) -> *m _PyObject_VectorcallTstate(tstate, func, args, nargsf, std::ptr::null_mut()) } -#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_9, not(PyPy)))] #[inline(always)] pub unsafe fn PyObject_CallMethodNoArgs( self_: *mut PyObject, @@ -191,7 +191,7 @@ pub unsafe fn PyObject_CallMethodNoArgs( ) } -#[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_9, not(PyPy)))] #[inline(always)] pub unsafe fn PyObject_CallMethodOneArg( self_: *mut PyObject, @@ -219,7 +219,7 @@ extern "C" { pub fn PyObject_LengthHint(o: *mut PyObject, arg1: Py_ssize_t) -> Py_ssize_t; #[cfg(not(Py_3_11))] // moved to src/buffer.rs from 3.11 - #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_9, not(PyPy)))] pub fn PyObject_CheckBuffer(obj: *mut PyObject) -> c_int; } diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index 306702de25e..9e233311ac1 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -1,6 +1,6 @@ use crate::object::*; use crate::Py_ssize_t; -#[cfg(not(any(PyPy, GraalPy, Py_LIMITED_API)))] +#[cfg(not(Py_LIMITED_API))] use std::os::raw::c_char; use std::os::raw::c_int; @@ -17,9 +17,18 @@ pub struct PyBytesObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyBytesObject); +opaque_struct!(pub PyBytesObject); extern "C" { #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")] pub fn _PyBytes_Resize(bytes: *mut *mut PyObject, newsize: Py_ssize_t) -> c_int; } + +#[cfg(not(Py_LIMITED_API))] +#[inline] +pub unsafe fn PyBytes_AS_STRING(op: *mut PyObject) -> *const c_char { + #[cfg(not(any(PyPy, GraalPy)))] + return &(*op.cast::()).ob_sval as *const c_char; + #[cfg(any(PyPy, GraalPy))] + return crate::PyBytes_AsString(op); +} diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 60989901052..3d47a1bc8c3 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -1,209 +1,43 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -#[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; +#[cfg(not(GraalPy))] +use std::os::raw::c_char; +use std::os::raw::{c_int, c_void}; #[cfg(not(any(PyPy, GraalPy)))] use std::ptr::addr_of_mut; -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped private _PY_MONITORING_LOCAL_EVENTS +// skipped private _PY_MONITORING_UNGROUPED_EVENTS +// skipped private _PY_MONITORING_EVENTS -#[cfg(Py_3_12)] -pub const _PY_MONITORING_LOCAL_EVENTS: usize = 10; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 15; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_EVENTS: usize = 17; +// skipped private _PyLocalMonitors +// skipped private _Py_GlobalMonitors -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_LocalMonitors { - pub tools: [u8; if cfg!(Py_3_13) { - _PY_MONITORING_LOCAL_EVENTS - } else { - _PY_MONITORING_UNGROUPED_EVENTS - }], -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_GlobalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], -} - -// skipped _Py_CODEUNIT +// skipped private _Py_CODEUNIT -// skipped _Py_OPCODE -// skipped _Py_OPARG +// skipped private _Py_OPCODE +// skipped private _Py_OPARG -// skipped _py_make_codeunit +// skipped private _py_make_codeunit -// skipped _py_set_opcode +// skipped private _py_set_opcode -// skipped _Py_MAKE_CODEUNIT -// skipped _Py_SET_OPCODE - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoCached { - pub _co_code: *mut PyObject, - pub _co_varnames: *mut PyObject, - pub _co_cellvars: *mut PyObject, - pub _co_freevars: *mut PyObject, -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoLineInstrumentationData { - pub original_opcode: u8, - pub line_delta: i8, -} +// skipped private _Py_MAKE_CODEUNIT +// skipped private _Py_SET_OPCODE -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoMonitoringData { - pub local_monitors: _Py_LocalMonitors, - pub active_monitors: _Py_LocalMonitors, - pub tools: *mut u8, - pub lines: *mut _PyCoLineInstrumentationData, - pub line_tools: *mut u8, - pub per_instruction_opcodes: *mut u8, - pub per_instruction_tools: *mut u8, -} - -#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))] -opaque_struct!(PyCodeObject); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_lnotab: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, -} - -#[cfg(Py_3_13)] -opaque_struct!(_PyExecutorArray); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - #[cfg(not(Py_3_10))] - pub co_lnotab: *mut PyObject, - #[cfg(Py_3_10)] - pub co_linetable: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, - pub co_opcache_map: *mut c_uchar, - pub co_opcache: *mut _PyOpcache, - pub co_opcache_flag: c_int, - pub co_opcache_size: c_uchar, -} +// skipped private _PyCoCached +// skipped private _PyCoLineInstrumentationData +// skipped private _PyCoMontoringData -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyVarObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_exceptiontable: *mut PyObject, - pub co_flags: c_int, - #[cfg(not(Py_3_12))] - pub co_warmup: c_int, +// skipped private _PyExecutorArray - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_stacksize: c_int, - pub co_firstlineno: c_int, - - pub co_nlocalsplus: c_int, - #[cfg(Py_3_12)] - pub co_framesize: c_int, - pub co_nlocals: c_int, - #[cfg(not(Py_3_12))] - pub co_nplaincellvars: c_int, - pub co_ncellvars: c_int, - pub co_nfreevars: c_int, - #[cfg(Py_3_12)] - pub co_version: u32, - - pub co_localsplusnames: *mut PyObject, - pub co_localspluskinds: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_qualname: *mut PyObject, - pub co_linetable: *mut PyObject, - pub co_weakreflist: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_code: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_linearray: *mut c_char, - #[cfg(Py_3_13)] - pub co_executors: *mut _PyExecutorArray, - #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, - #[cfg(all(Py_3_12, not(Py_3_13)))] - pub _co_instrumentation_version: u64, - #[cfg(Py_3_13)] - pub _co_instrumentation_version: libc::uintptr_t, - #[cfg(Py_3_12)] - pub _co_monitoring: *mut _PyCoMonitoringData, - pub _co_firsttraceable: c_int, - pub co_extra: *mut c_void, - pub co_code_adaptive: [c_char; 1], -} - -#[cfg(PyPy)] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_name: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_argcount: c_int, - pub co_flags: c_int, -} +opaque_struct!( + #[doc = "A Python code object.\n"] + #[doc = "\n"] + #[doc = "`pyo3-ffi` does not expose the contents of this struct, as it has no stability guarantees."] + pub PyCodeObject +); /* Masks for co_flags */ pub const CO_OPTIMIZED: c_int = 0x0001; @@ -249,28 +83,14 @@ pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyCode_Type)) as c_int } -#[inline] -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10, not(Py_3_11)))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { - crate::PyTuple_GET_SIZE((*op).co_freevars) -} - -#[inline] -#[cfg(all(not(Py_3_10), Py_3_11, not(any(PyPy, GraalPy))))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> c_int { - (*op).co_nfreevars -} - extern "C" { #[cfg(PyPy)] #[link_name = "PyPyCode_Check"] pub fn PyCode_Check(op: *mut PyObject) -> c_int; - - #[cfg(PyPy)] - #[link_name = "PyPyCode_GetNumFree"] - pub fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t; } +// skipped PyCode_GetNumFree (requires knowledge of code object layout) + extern "C" { #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_New")] diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 79f06c92003..078b5e0d675 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -6,19 +6,21 @@ use crate::pyarena::*; use crate::pythonrun::*; #[cfg(not(any(PyPy, Py_3_10)))] use crate::PyCodeObject; +use crate::INT_MAX; #[cfg(not(any(PyPy, Py_3_10)))] use std::os::raw::c_char; use std::os::raw::c_int; -// skipped non-limited PyCF_MASK -// skipped non-limited PyCF_MASK_OBSOLETE -// skipped non-limited PyCF_SOURCE_IS_UTF8 -// skipped non-limited PyCF_DONT_IMPLY_DEDENT -// skipped non-limited PyCF_ONLY_AST -// skipped non-limited PyCF_IGNORE_COOKIE -// skipped non-limited PyCF_TYPE_COMMENTS -// skipped non-limited PyCF_ALLOW_TOP_LEVEL_AWAIT -// skipped non-limited PyCF_COMPILE_MASK +// skipped PyCF_MASK +// skipped PyCF_MASK_OBSOLETE +// skipped PyCF_SOURCE_IS_UTF8 +// skipped PyCF_DONT_IMPLY_DEDENT +// skipped PyCF_ONLY_AST +// skipped PyCF_IGNORE_COOKIE +// skipped PyCF_TYPE_COMMENTS +// skipped PyCF_ALLOW_TOP_LEVEL_AWAIT +// skipped PyCF_OPTIMIZED_AST +// skipped PyCF_COMPILE_MASK #[repr(C)] #[derive(Copy, Clone)] @@ -28,31 +30,23 @@ pub struct PyCompilerFlags { pub cf_feature_version: c_int, } -// skipped non-limited _PyCompilerFlags_INIT +// skipped _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCompilerSrcLocation { - pub lineno: c_int, - pub end_lineno: c_int, - pub col_offset: c_int, - pub end_col_offset: c_int, -} - -// skipped SRC_LOCATION_FROM_AST - -#[cfg(not(any(PyPy, GraalPy, Py_3_13)))] +// NB this type technically existed in the header until 3.13, when it was +// moved to the internal CPython headers. +// +// We choose not to expose it in the public API past 3.10, as it is +// not used in the public API past that point. +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, - #[cfg(not(Py_3_12))] pub ff_lineno: c_int, - #[cfg(Py_3_12)] - pub ff_location: _PyCompilerSrcLocation, } +// FIXME: these constants should probably be &CStr, if they are used at all + pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; pub const FUTURE_GENERATORS: &str = "generators"; pub const FUTURE_DIVISION: &str = "division"; @@ -62,13 +56,12 @@ pub const FUTURE_PRINT_FUNCTION: &str = "print_function"; pub const FUTURE_UNICODE_LITERALS: &str = "unicode_literals"; pub const FUTURE_BARRY_AS_BDFL: &str = "barry_as_FLUFL"; pub const FUTURE_GENERATOR_STOP: &str = "generator_stop"; -// skipped non-limited FUTURE_ANNOTATIONS +pub const FUTURE_ANNOTATIONS: &str = "annotations"; +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyNode_Compile(arg1: *mut _node, arg2: *const c_char) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileEx( _mod: *mut _mod, filename: *const c_char, @@ -77,7 +70,6 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileObject( _mod: *mut _mod, filename: *mut PyObject, @@ -86,23 +78,20 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromAST(_mod: *mut _mod, filename: *const c_char) -> *mut PyFutureFeatures; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromASTObject( _mod: *mut _mod, filename: *mut PyObject, ) -> *mut PyFutureFeatures; +} - // skipped non-limited _Py_Mangle - // skipped non-limited PY_INVALID_STACK_EFFECT +pub const PY_INVALID_STACK_EFFECT: c_int = INT_MAX; + +extern "C" { pub fn PyCompile_OpcodeStackEffect(opcode: c_int, oparg: c_int) -> c_int; #[cfg(Py_3_8)] pub fn PyCompile_OpcodeStackEffectWithJump(opcode: c_int, oparg: c_int, jump: c_int) -> c_int; - - // skipped non-limited _PyASTOptimizeState - // skipped non-limited _PyAST_Optimize } diff --git a/pyo3-ffi/src/cpython/critical_section.rs b/pyo3-ffi/src/cpython/critical_section.rs index 97b2f5e0559..808dba870c6 100644 --- a/pyo3-ffi/src/cpython/critical_section.rs +++ b/pyo3-ffi/src/cpython/critical_section.rs @@ -17,10 +17,10 @@ pub struct PyCriticalSection2 { } #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection); +opaque_struct!(pub PyCriticalSection); #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection2); +opaque_struct!(pub PyCriticalSection2); extern "C" { pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject); diff --git a/pyo3-ffi/src/cpython/descrobject.rs b/pyo3-ffi/src/cpython/descrobject.rs index 1b5ee466c8e..7cef9bdbf42 100644 --- a/pyo3-ffi/src/cpython/descrobject.rs +++ b/pyo3-ffi/src/cpython/descrobject.rs @@ -69,10 +69,7 @@ pub struct PyWrapperDescrObject { pub d_wrapped: *mut c_void, } -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PyMethodWrapper_Type: PyTypeObject; -} +// skipped _PyMethodWrapper_Type // skipped non-limited PyDescr_NewWrapper // skipped non-limited PyDescr_IsData diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 79dcbfdb62e..34b66a9699d 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -2,10 +2,10 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use std::os::raw::c_int; -opaque_struct!(PyDictKeysObject); +opaque_struct!(pub PyDictKeysObject); #[cfg(Py_3_11)] -opaque_struct!(PyDictValues); +opaque_struct!(pub PyDictValues); #[cfg(not(GraalPy))] #[repr(C)] @@ -17,7 +17,10 @@ pub struct PyDictObject { Py_3_12, deprecated(note = "Deprecated in Python 3.12 and will be removed in the future.") )] + #[cfg(not(Py_3_14))] pub ma_version_tag: u64, + #[cfg(Py_3_14)] + _ma_watcher_tag: u64, pub ma_keys: *mut PyDictKeysObject, #[cfg(not(Py_3_11))] pub ma_values: *mut *mut PyObject, diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index e9b9c183f37..993e93c838b 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -53,7 +53,7 @@ pub struct PyFrameObject { } #[cfg(any(PyPy, GraalPy, Py_3_11))] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); // skipped _PyFrame_IsRunnable // skipped _PyFrame_IsExecuting diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 25de30d57f7..cd2052de174 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -41,6 +41,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_14)] + pub func_annotate: *mut PyObject, #[cfg(Py_3_12)] pub func_typeparams: *mut PyObject, pub vectorcall: Option, diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index c9d419e3782..92f14d59e4b 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -1,13 +1,11 @@ use crate::object::*; use crate::PyFrameObject; -#[cfg(not(any(PyPy, GraalPy)))] -use crate::_PyErr_StackItem; -#[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_11, not(any(PyPy, GraalPy, Py_3_14))))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_14)))] #[repr(C)] pub struct PyGenObject { pub ob_base: PyObject, @@ -20,7 +18,7 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - pub gi_exc_state: _PyErr_StackItem, + pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, #[cfg(Py_3_11)] @@ -35,6 +33,9 @@ pub struct PyGenObject { pub gi_iframe: [*mut PyObject; 1], } +#[cfg(all(Py_3_14, not(any(PyPy, GraalPy))))] +opaque_struct!(pub PyGenObject); + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyGen_Type: PyTypeObject; @@ -67,9 +68,10 @@ extern "C" { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCoro_Type: PyTypeObject; - pub static mut _PyCoroWrapper_Type: PyTypeObject; } +// skipped _PyCoroWrapper_Type + #[inline] pub unsafe fn PyCoro_CheckExact(op: *mut PyObject) -> c_int { PyObject_TypeCheck(op, addr_of_mut!(PyCoro_Type)) diff --git a/pyo3-ffi/src/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs index 697d68a419c..c8ef5ab487e 100644 --- a/pyo3-ffi/src/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -65,10 +65,8 @@ pub struct _frozen { extern "C" { #[cfg(not(PyPy))] pub static mut PyImport_FrozenModules: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenBootstrap: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenStdlib: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenTest: *const _frozen; } + +// skipped _PyImport_FrozenBootstrap +// skipped _PyImport_FrozenStdlib +// skipped _PyImport_FrozenTest diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 321d200e141..076981f1485 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -93,6 +93,8 @@ pub struct PyConfig { pub tracemalloc: c_int, #[cfg(Py_3_12)] pub perf_profiling: c_int, + #[cfg(Py_3_14)] + pub remote_debug: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -141,10 +143,18 @@ pub struct PyConfig { pub safe_path: c_int, #[cfg(Py_3_12)] pub int_max_str_digits: c_int, + #[cfg(Py_3_14)] + pub thread_inherit_context: c_int, + #[cfg(Py_3_14)] + pub context_aware_warnings: c_int, + #[cfg(all(Py_3_14, target_os = "macos"))] + pub use_system_logger: c_int, #[cfg(Py_3_13)] pub cpu_count: c_int, #[cfg(Py_GIL_DISABLED)] pub enable_gil: c_int, + #[cfg(all(Py_3_14, Py_GIL_DISABLED))] + pub tlbc_enabled: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index f09d51d0e4e..adaf8bc8861 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -38,6 +38,7 @@ pub(crate) mod pythonrun; // skipped sysmodule.h pub(crate) mod floatobject; pub(crate) mod pyframe; +pub(crate) mod pyhash; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; pub(crate) mod weakrefobject; @@ -73,6 +74,8 @@ pub use self::pydebug::*; pub use self::pyerrors::*; #[cfg(all(Py_3_11, not(PyPy)))] pub use self::pyframe::*; +#[cfg(any(not(PyPy), Py_3_13))] +pub use self::pyhash::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; pub use self::pymem::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 4e6932da789..26ef784dde1 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -312,8 +312,12 @@ pub struct PyHeapTypeObject { pub ht_module: *mut object::PyObject, #[cfg(all(Py_3_11, not(PyPy)))] _ht_tpname: *mut c_char, + #[cfg(Py_3_14)] + pub ht_token: *mut c_void, #[cfg(all(Py_3_11, not(PyPy)))] _spec_cache: _specialization_cache, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub unique_id: Py_ssize_t, } impl Default for PyHeapTypeObject { diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index c6e10e5f07b..c9831669ac7 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -46,6 +46,8 @@ pub struct PySyntaxErrorObject { pub end_offset: *mut PyObject, pub text: *mut PyObject, pub print_file_and_line: *mut PyObject, + #[cfg(Py_3_14)] + pub metadata: *mut PyObject, } #[cfg(not(any(PyPy, GraalPy)))] diff --git a/pyo3-ffi/src/cpython/pyframe.rs b/pyo3-ffi/src/cpython/pyframe.rs index 5e1e16a7d08..f0c38be47be 100644 --- a/pyo3-ffi/src/cpython/pyframe.rs +++ b/pyo3-ffi/src/cpython/pyframe.rs @@ -1,2 +1,3 @@ +// NB used in `_PyEval_EvalFrameDefault`, maybe we remove this too. #[cfg(all(Py_3_11, not(PyPy)))] -opaque_struct!(_PyInterpreterFrame); +opaque_struct!(pub _PyInterpreterFrame); diff --git a/pyo3-ffi/src/cpython/pyhash.rs b/pyo3-ffi/src/cpython/pyhash.rs new file mode 100644 index 00000000000..b746018a38a --- /dev/null +++ b/pyo3-ffi/src/cpython/pyhash.rs @@ -0,0 +1,38 @@ +#[cfg(Py_3_14)] +use crate::Py_ssize_t; +#[cfg(Py_3_13)] +use crate::{PyObject, Py_hash_t}; +#[cfg(any(Py_3_13, not(PyPy)))] +use std::os::raw::c_void; +#[cfg(not(PyPy))] +use std::os::raw::{c_char, c_int}; + +#[cfg(not(PyPy))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct PyHash_FuncDef { + pub hash: + Option crate::Py_hash_t>, + pub name: *const c_char, + pub hash_bits: c_int, + pub seed_bits: c_int, +} + +#[cfg(not(PyPy))] +impl Default for PyHash_FuncDef { + #[inline] + fn default() -> Self { + unsafe { std::mem::zeroed() } + } +} + +extern "C" { + #[cfg(not(PyPy))] + pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; + #[cfg(Py_3_13)] + pub fn Py_HashPointer(ptr: *const c_void) -> Py_hash_t; + #[cfg(Py_3_13)] + pub fn PyObject_GenericHash(obj: *mut PyObject) -> Py_hash_t; + #[cfg(Py_3_14)] + pub fn Py_HashBuffer(ptr: *const c_void, len: Py_ssize_t) -> Py_hash_t; +} diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 650cd6a1f7f..b8f6fd667b7 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -27,16 +27,18 @@ pub const PyTrace_OPCODE: c_int = 7; // skipped PyTraceInfo // skipped CFrame +/// Private structure used inline in `PyGenObject` #[cfg(not(PyPy))] #[repr(C)] #[derive(Clone, Copy)] +#[doc(hidden)] // TODO should be able to make pub(crate) after MSRV 1.74 pub struct _PyErr_StackItem { #[cfg(not(Py_3_11))] - pub exc_type: *mut PyObject, - pub exc_value: *mut PyObject, + exc_type: *mut PyObject, + exc_value: *mut PyObject, #[cfg(not(Py_3_11))] - pub exc_traceback: *mut PyObject, - pub previous_item: *mut _PyErr_StackItem, + exc_traceback: *mut PyObject, + previous_item: *mut _PyErr_StackItem, } // skipped _PyStackChunk diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 9616d4372cc..dc1bf8e40d0 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -1,10 +1,14 @@ use crate::object::*; +#[cfg(Py_3_14)] +use crate::pyport::Py_hash_t; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, + #[cfg(Py_3_14)] + pub ob_hash: Py_hash_t, pub ob_item: [*mut PyObject; 1], } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 3527a5aeadb..452c82e4c4b 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -31,11 +31,13 @@ use std::os::raw::{c_char, c_int, c_uint, c_void}; // skipped Py_UNICODE_LOW_SURROGATE // generated by bindgen v0.63.0 (with small adaptations) +#[cfg(not(Py_3_14))] #[repr(C)] struct BitfieldUnit { storage: Storage, } +#[cfg(not(Py_3_14))] impl BitfieldUnit { #[inline] pub const fn new(storage: Storage) -> Self { @@ -43,7 +45,7 @@ impl BitfieldUnit { } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] impl BitfieldUnit where Storage: AsRef<[u8]> + AsMut<[u8]>, @@ -117,27 +119,33 @@ where } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_INDEX: usize = 0; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_WIDTH: u8 = 2; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_WIDTH: u8 = 3; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH) as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_WIDTH: u8 = 1; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH) as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_WIDTH: u8 = 1; +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_INDEX: usize = + (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_WIDTH: u8 = 1; + #[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; @@ -153,15 +161,15 @@ const STATE_READY_WIDTH: u8 = 1; /// /// Memory layout of C bitfields is implementation defined, so these functions are still /// unsafe. Users must verify that they work as expected on the architectures they target. +#[cfg(not(Py_3_14))] #[repr(C)] -#[repr(align(4))] struct PyASCIIObjectState { bitfield_align: [u8; 0], bitfield: BitfieldUnit<[u8; 4usize]>, } // c_uint and u32 are not necessarily the same type on all targets / architectures -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[allow(clippy::useless_transmute)] impl PyASCIIObjectState { #[inline] @@ -215,6 +223,26 @@ impl PyASCIIObjectState { .set(STATE_ASCII_INDEX, STATE_ASCII_WIDTH, val as u64) } + #[cfg(Py_3_12)] + #[inline] + unsafe fn statically_allocated(&self) -> c_uint { + std::mem::transmute(self.bitfield.get( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + ) as u32) + } + + #[cfg(Py_3_12)] + #[inline] + unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let val: u32 = std::mem::transmute(val); + self.bitfield.set( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + val as u64, + ) + } + #[cfg(not(Py_3_12))] #[inline] unsafe fn ready(&self) -> c_uint { @@ -230,6 +258,7 @@ impl PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for PyASCIIObjectState { #[inline] fn from(value: u32) -> Self { @@ -240,6 +269,7 @@ impl From for PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for u32 { #[inline] fn from(value: PyASCIIObjectState) -> Self { @@ -258,19 +288,29 @@ pub struct PyASCIIObject { /// Rust doesn't expose bitfields. So we have accessor functions for /// retrieving values. /// + /// Before 3.12: /// unsigned int interned:2; // SSTATE_* constants. /// unsigned int kind:3; // PyUnicode_*_KIND constants. /// unsigned int compact:1; /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; + /// + /// 3.12, and 3.13 + /// unsigned int interned:2; // SSTATE_* constants. + /// unsigned int kind:3; // PyUnicode_*_KIND constants. + /// unsigned int compact:1; + /// unsigned int ascii:1; + /// unsigned int statically_allocated:1; + /// unsigned int :24; + /// on 3.14 and higher PyO3 doesn't access the internal state pub state: u32, #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] impl PyASCIIObject { #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. @@ -347,6 +387,7 @@ impl PyASCIIObject { /// /// Calling this function with an argument that is neither `0` nor `1` is invalid. #[inline] + #[cfg(not(all(Py_3_14, Py_GIL_DISABLED)))] pub unsafe fn set_ascii(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); state.set_ascii(val); @@ -372,6 +413,26 @@ impl PyASCIIObject { state.set_ready(val); self.state = u32::from(state); } + + /// Get the `statically_allocated` field of the [`PyASCIIObject`] state bitfield. + /// + /// Returns either `0` or `1`. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn statically_allocated(&self) -> c_uint { + PyASCIIObjectState::from(self.state).statically_allocated() + } + + /// Set the `statically_allocated` flag of the [`PyASCIIObject`] state bitfield. + /// + /// Calling this function with an argument that is neither `0` nor `1` is invalid. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let mut state = PyASCIIObjectState::from(self.state); + state.set_statically_allocated(val); + self.state = u32::from(state); + } } #[repr(C)] @@ -413,7 +474,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; #[cfg(Py_3_12)] pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -423,13 +484,13 @@ pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ascii() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).compact() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { ((*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0).into() @@ -461,7 +522,13 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 } -#[cfg(not(GraalPy))] +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_KIND")] + pub fn PyUnicode_KIND(op: *mut PyObject) -> c_uint; +} + +#[cfg(all(not(GraalPy), not(Py_3_14)))] #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -471,7 +538,7 @@ pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).kind() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { if PyUnicode_IS_ASCII(op) != 0 { @@ -489,7 +556,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } -#[cfg(not(any(GraalPy, PyPy)))] +#[cfg(not(any(GraalPy, PyPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -501,6 +568,13 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { } } +#[cfg(Py_3_14)] +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_DATA")] + pub fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void; +} + // skipped PyUnicode_WRITE // skipped PyUnicode_READ // skipped PyUnicode_READ_CHAR diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 88bb501bcc5..1c50c7a759f 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -1,3 +1,4 @@ +// NB publicly re-exported in `src/weakrefobject.rs` #[cfg(not(any(PyPy, GraalPy)))] pub struct _PyWeakReference { pub ob_base: crate::PyObject, diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 7f2d7958364..4db1bdd16d1 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -487,7 +487,9 @@ extern "C" { pub fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_GET_FOLD"] pub fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_DATE_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_DATE_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_TIME_GET_HOUR"] pub fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int; @@ -499,7 +501,9 @@ extern "C" { pub fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_TIME_GET_FOLD"] pub fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_TIME_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_TIME_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_DELTA_GET_DAYS"] pub fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 710be80243f..e609352ac1b 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -118,4 +118,4 @@ extern "C" { #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyDictObject); +opaque_struct!(pub PyDictObject); diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 65fc1d4c316..4e1d6476f58 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -4,7 +4,7 @@ use std::ptr::addr_of_mut; #[cfg(Py_LIMITED_API)] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyFloatObject); +opaque_struct!(pub PyFloatObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index aba8866b266..f78c918a8c5 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -341,9 +341,10 @@ // model opaque types: // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs macro_rules! opaque_struct { - ($name:ident) => { + ($(#[$attrs:meta])* $pub:vis $name:ident) => { + $(#[$attrs])* #[repr(C)] - pub struct $name([u8; 0]); + $pub struct $name([u8; 0]); }; } @@ -447,6 +448,7 @@ pub use self::pystate::*; pub use self::pystrtod::*; pub use self::pythonrun::*; pub use self::rangeobject::*; +pub use self::refcount::*; pub use self::setobject::*; pub use self::sliceobject::*; pub use self::structseq::*; @@ -539,6 +541,7 @@ mod pystrtod; // skipped pythread.h // skipped pytime.h mod rangeobject; +mod refcount; mod setobject; mod sliceobject; mod structseq; diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 68b4ecba540..eca0af3d0a5 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -4,7 +4,7 @@ use libc::size_t; use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; -opaque_struct!(PyLongObject); +opaque_struct!(pub PyLongObject); #[inline] pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { diff --git a/pyo3-ffi/src/memoryobject.rs b/pyo3-ffi/src/memoryobject.rs index b7ef9e2ef1d..4e1e50c6c82 100644 --- a/pyo3-ffi/src/memoryobject.rs +++ b/pyo3-ffi/src/memoryobject.rs @@ -3,11 +3,10 @@ use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; use std::ptr::addr_of_mut; +// skipped _PyManagedBuffer_Type + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { - #[cfg(not(Py_LIMITED_API))] - pub static mut _PyManagedBuffer_Type: PyTypeObject; - #[cfg_attr(PyPy, link_name = "PyPyMemoryView_Type")] pub static mut PyMemoryView_Type: PyTypeObject; } diff --git a/pyo3-ffi/src/modsupport.rs b/pyo3-ffi/src/modsupport.rs index 4a18d30f97c..56a68fe2613 100644 --- a/pyo3-ffi/src/modsupport.rs +++ b/pyo3-ffi/src/modsupport.rs @@ -36,6 +36,8 @@ extern "C" { pub fn Py_BuildValue(arg1: *const c_char, ...) -> *mut PyObject; // skipped Py_VaBuildValue + #[cfg(Py_3_13)] + pub fn PyModule_Add(module: *mut PyObject, name: *const c_char, value: *mut PyObject) -> c_int; #[cfg(Py_3_10)] #[cfg_attr(PyPy, link_name = "PyPyModule_AddObjectRef")] pub fn PyModule_AddObjectRef( diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 087cd32920c..c4c5abc361c 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,5 +1,7 @@ use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(Py_GIL_DISABLED)] +use crate::refcount; +#[cfg(Py_GIL_DISABLED)] use crate::PyMutex; #[cfg(Py_GIL_DISABLED)] use std::marker::PhantomPinned; @@ -7,81 +9,50 @@ use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; use std::ptr; #[cfg(Py_GIL_DISABLED)] -use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8, Ordering::Relaxed}; +use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8}; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyTypeObject); +opaque_struct!(pub PyTypeObject); #[cfg(not(Py_LIMITED_API))] pub use crate::cpython::object::PyTypeObject; -#[cfg(Py_3_12)] -const _Py_IMMORTAL_REFCNT: Py_ssize_t = { - if cfg!(target_pointer_width = "64") { - c_uint::MAX as Py_ssize_t - } else { - // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) - (c_uint::MAX >> 2) as Py_ssize_t - } -}; - -#[cfg(Py_GIL_DISABLED)] -const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; - -#[allow(clippy::declare_interior_mutable_const)] -pub const PyObject_HEAD_INIT: PyObject = PyObject { - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_next: std::ptr::null_mut(), - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_prev: std::ptr::null_mut(), - #[cfg(Py_GIL_DISABLED)] - ob_tid: 0, - #[cfg(Py_GIL_DISABLED)] - _padding: 0, - #[cfg(Py_GIL_DISABLED)] - ob_mutex: PyMutex { - _bits: AtomicU8::new(0), - _pin: PhantomPinned, - }, - #[cfg(Py_GIL_DISABLED)] - ob_gc_bits: 0, - #[cfg(Py_GIL_DISABLED)] - ob_ref_local: AtomicU32::new(_Py_IMMORTAL_REFCNT_LOCAL), - #[cfg(Py_GIL_DISABLED)] - ob_ref_shared: AtomicIsize::new(0), - #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] - ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, - #[cfg(not(Py_3_12))] - ob_refcnt: 1, - #[cfg(PyPy)] - ob_pypy_link: 0, - ob_type: std::ptr::null_mut(), -}; - -// skipped PyObject_VAR_HEAD -// skipped Py_INVALID_SIZE - -// skipped private _Py_UNOWNED_TID +// skip PyObject_HEAD -#[cfg(Py_GIL_DISABLED)] -const _Py_REF_SHARED_SHIFT: isize = 2; -// skipped private _Py_REF_SHARED_FLAG_MASK - -// skipped private _Py_REF_SHARED_INIT -// skipped private _Py_REF_MAYBE_WEAKREF -// skipped private _Py_REF_QUEUED -// skipped private _Py_REF_MERGED +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED), target_endian = "big"))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_flags: u16, + pub ob_overflow: u16, + pub ob_refcnt: u32, +} -// skipped private _Py_REF_SHARED +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED), target_endian = "little"))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_refcnt: u32, + pub ob_overflow: u16, + pub ob_flags: u16, +} #[repr(C)] #[derive(Copy, Clone)] #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] /// This union is anonymous in CPython, so the name was given by PyO3 because -/// Rust unions need a name. +/// Rust union need a name. pub union PyObjectObRefcnt { + #[cfg(all(target_pointer_width = "64", Py_3_14))] + pub ob_refcnt_full: crate::PY_INT64_T, + #[cfg(Py_3_14)] + pub refcnt_and_flags: PyObjectObFlagsAndRefcnt, pub ob_refcnt: Py_ssize_t, - #[cfg(target_pointer_width = "64")] + #[cfg(all(target_pointer_width = "64", not(Py_3_14)))] pub ob_refcnt_split: [crate::PY_UINT32_T; 2], } @@ -95,6 +66,9 @@ impl std::fmt::Debug for PyObjectObRefcnt { #[cfg(all(not(Py_3_12), not(Py_GIL_DISABLED)))] pub type PyObjectObRefcnt = Py_ssize_t; +// PyObject_HEAD_INIT comes before the PyObject definition in object.h +// but we put it after PyObject because HEAD_INIT uses PyObject + #[repr(C)] #[derive(Debug)] pub struct PyObject { @@ -104,8 +78,10 @@ pub struct PyObject { pub _ob_prev: *mut PyObject, #[cfg(Py_GIL_DISABLED)] pub ob_tid: libc::uintptr_t, - #[cfg(Py_GIL_DISABLED)] + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] pub _padding: u16, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub ob_flags: u16, #[cfg(Py_GIL_DISABLED)] pub ob_mutex: PyMutex, // per-object lock #[cfg(Py_GIL_DISABLED)] @@ -121,7 +97,41 @@ pub struct PyObject { pub ob_type: *mut PyTypeObject, } -// skipped private _PyObject_CAST +#[allow(clippy::declare_interior_mutable_const)] +pub const PyObject_HEAD_INIT: PyObject = PyObject { + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_next: std::ptr::null_mut(), + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_prev: std::ptr::null_mut(), + #[cfg(Py_GIL_DISABLED)] + ob_tid: 0, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + ob_flags: 0, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + _padding: 0, + #[cfg(Py_GIL_DISABLED)] + ob_mutex: PyMutex { + _bits: AtomicU8::new(0), + _pin: PhantomPinned, + }, + #[cfg(Py_GIL_DISABLED)] + ob_gc_bits: 0, + #[cfg(Py_GIL_DISABLED)] + ob_ref_local: AtomicU32::new(refcount::_Py_IMMORTAL_REFCNT_LOCAL), + #[cfg(Py_GIL_DISABLED)] + ob_ref_shared: AtomicIsize::new(0), + #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] + ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, + #[cfg(not(Py_3_12))] + ob_refcnt: 1, + #[cfg(PyPy)] + ob_pypy_link: 0, + ob_type: std::ptr::null_mut(), +}; + +// skipped _Py_UNOWNED_TID + +// skipped _PyObject_CAST #[repr(C)] #[derive(Debug)] @@ -137,54 +147,36 @@ pub struct PyVarObject { // skipped private _PyVarObject_CAST #[inline] -#[cfg(not(all(PyPy, Py_3_10)))] +#[cfg(not(any(GraalPy, all(PyPy, Py_3_10))))] #[cfg_attr(docsrs, doc(cfg(all())))] pub unsafe fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int { (x == y).into() } -#[cfg(all(PyPy, Py_3_10))] +#[cfg(any(GraalPy, all(PyPy, Py_3_10)))] #[cfg_attr(docsrs, doc(cfg(all())))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPy_Is")] pub fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int; } -// skipped private _Py_GetThreadLocal_Addr +// skipped _Py_GetThreadLocal_Addr -// skipped private _Py_ThreadId +// skipped _Py_ThreadID -// skipped private _Py_IsOwnedByCurrentThread +// skipped _Py_IsOwnedByCurrentThread -#[inline] -pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - #[cfg(Py_GIL_DISABLED)] - { - let local = (*ob).ob_ref_local.load(Relaxed); - if local == _Py_IMMORTAL_REFCNT_LOCAL { - return _Py_IMMORTAL_REFCNT; - } - let shared = (*ob).ob_ref_shared.load(Relaxed); - local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) - } - - #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] - { - (*ob).ob_refcnt.ob_refcnt - } - - #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] - { - (*ob).ob_refcnt - } +#[cfg(GraalPy)] +extern "C" { + #[cfg(GraalPy)] + fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; - #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] - { - _Py_REFCNT(ob) - } + #[cfg(GraalPy)] + fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; } #[inline] +#[cfg(not(Py_3_14))] pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { #[cfg(not(GraalPy))] return (*ob).ob_type; @@ -192,6 +184,15 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { return _Py_TYPE(ob); } +#[cfg_attr(windows, link(name = "pythonXY"))] +#[cfg(Py_3_14)] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPy_TYPE")] + pub fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject; +} + +// skip _Py_TYPE compat shim + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] @@ -212,29 +213,11 @@ pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { _Py_SIZE(ob) } -#[inline(always)] -#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] -unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - #[cfg(target_pointer_width = "64")] - { - (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int - } - - #[cfg(target_pointer_width = "32")] - { - ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int - } -} - #[inline] pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } -// skipped _Py_SetRefCnt - -// skipped Py_SET_REFCNT - // skipped Py_SET_TYPE // skipped Py_SET_SIZE @@ -586,222 +569,6 @@ pub const Py_TPFLAGS_DEFAULT: c_ulong = if cfg!(Py_3_10) { pub const Py_TPFLAGS_HAVE_FINALIZE: c_ulong = 1; pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -extern "C" { - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_INCREF_IncRefTotal(); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_DECREF_DecRefTotal(); - - #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] - fn _Py_Dealloc(arg1: *mut PyObject); - - #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] - #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] - pub fn Py_IncRef(o: *mut PyObject); - #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] - #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] - pub fn Py_DecRef(o: *mut PyObject); - - #[cfg(all(Py_3_10, not(PyPy)))] - fn _Py_IncRef(o: *mut PyObject); - #[cfg(all(Py_3_10, not(PyPy)))] - fn _Py_DecRef(o: *mut PyObject); - - #[cfg(GraalPy)] - fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; - - #[cfg(GraalPy)] - fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; - - #[cfg(GraalPy)] - fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; -} - -#[inline(always)] -pub unsafe fn Py_INCREF(op: *mut PyObject) { - // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting - // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. - #[cfg(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - py_sys_config = "Py_REF_DEBUG", - GraalPy - ))] - { - // _Py_IncRef was added to the ABI in 3.10; skips null checks - #[cfg(all(Py_3_10, not(PyPy)))] - { - _Py_IncRef(op); - } - - #[cfg(any(not(Py_3_10), PyPy))] - { - Py_IncRef(op); - } - } - - // version-specific builds are allowed to directly manipulate the reference count - #[cfg(not(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - py_sys_config = "Py_REF_DEBUG", - GraalPy - )))] - { - #[cfg(all(Py_3_12, target_pointer_width = "64"))] - { - let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; - let new_refcnt = cur_refcnt.wrapping_add(1); - if new_refcnt == 0 { - return; - } - (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; - } - - #[cfg(all(Py_3_12, target_pointer_width = "32"))] - { - if _Py_IsImmortal(op) != 0 { - return; - } - (*op).ob_refcnt.ob_refcnt += 1 - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt += 1 - } - - // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - } -} - -#[inline(always)] -#[cfg_attr( - all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), - track_caller -)] -pub unsafe fn Py_DECREF(op: *mut PyObject) { - // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting - // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts - // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. - #[cfg(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), - GraalPy - ))] - { - // _Py_DecRef was added to the ABI in 3.10; skips null checks - #[cfg(all(Py_3_10, not(PyPy)))] - { - _Py_DecRef(op); - } - - #[cfg(any(not(Py_3_10), PyPy))] - { - Py_DecRef(op); - } - } - - #[cfg(not(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), - GraalPy - )))] - { - #[cfg(Py_3_12)] - if _Py_IsImmortal(op) != 0 { - return; - } - - // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - - #[cfg(py_sys_config = "Py_REF_DEBUG")] - _Py_DECREF_DecRefTotal(); - - #[cfg(Py_3_12)] - { - (*op).ob_refcnt.ob_refcnt -= 1; - - #[cfg(py_sys_config = "Py_REF_DEBUG")] - if (*op).ob_refcnt.ob_refcnt < 0 { - let location = std::panic::Location::caller(); - let filename = std::ffi::CString::new(location.file()).unwrap(); - _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); - } - - if (*op).ob_refcnt.ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt -= 1; - - if (*op).ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - } -} - -#[inline] -pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { - let tmp = *op; - if !tmp.is_null() { - *op = ptr::null_mut(); - Py_DECREF(tmp); - } -} - -#[inline] -pub unsafe fn Py_XINCREF(op: *mut PyObject) { - if !op.is_null() { - Py_INCREF(op) - } -} - -#[inline] -pub unsafe fn Py_XDECREF(op: *mut PyObject) { - if !op.is_null() { - Py_DECREF(op) - } -} - -extern "C" { - #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] - #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] - pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] - #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] - pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; -} - -// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here -// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here - -#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] -#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] -#[inline] -pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { - Py_INCREF(obj); - obj -} - -#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] -#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] -#[inline] -pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { - Py_XINCREF(obj); - obj -} - #[cfg(Py_3_13)] pub const Py_CONSTANT_NONE: c_uint = 0; #[cfg(Py_3_13)] @@ -942,4 +709,7 @@ extern "C" { arg1: *mut crate::PyTypeObject, arg2: *mut crate::PyModuleDef, ) -> *mut PyObject; + + #[cfg(Py_3_14)] + pub fn PyType_Freeze(tp: *mut crate::PyTypeObject) -> c_int; } diff --git a/pyo3-ffi/src/pyarena.rs b/pyo3-ffi/src/pyarena.rs index 87d5f28a7a5..1200de3df48 100644 --- a/pyo3-ffi/src/pyarena.rs +++ b/pyo3-ffi/src/pyarena.rs @@ -1 +1 @@ -opaque_struct!(PyArena); +opaque_struct!(pub PyArena); diff --git a/pyo3-ffi/src/pyframe.rs b/pyo3-ffi/src/pyframe.rs index 4dd3d2b31a5..1693b20b0af 100644 --- a/pyo3-ffi/src/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -6,7 +6,7 @@ use crate::PyFrameObject; use std::os::raw::c_int; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); extern "C" { pub fn PyFrame_GetLineNumber(f: *mut PyFrameObject) -> c_int; diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index 4f14e04a695..074278ddf3c 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,5 @@ #[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::c_char; #[cfg(not(any(Py_LIMITED_API, PyPy)))] use std::os::raw::c_void; @@ -22,29 +20,6 @@ pub const _PyHASH_MULTIPLIER: c_ulong = 1000003; // skipped non-limited _Py_HashSecret_t -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyHash_FuncDef { - pub hash: Option Py_hash_t>, - pub name: *const c_char, - pub hash_bits: c_int, - pub seed_bits: c_int, -} - -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -impl Default for PyHash_FuncDef { - #[inline] - fn default() -> Self { - unsafe { std::mem::zeroed() } - } -} - -extern "C" { - #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] - pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; -} - // skipped Py_HASH_CUTOFF pub const Py_HASH_EXTERNAL: c_int = 0; diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index a144c67fb1b..e524831d80d 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,8 @@ +// NB libc does not define this constant on all platforms, so we hard code it +// like CPython does. +// https://github.com/python/cpython/blob/d8b9011702443bb57579f8834f3effe58e290dfc/Include/pyport.h#L372 +pub const INT_MAX: std::os::raw::c_int = 2147483647; + pub type PY_UINT32_T = u32; pub type PY_UINT64_T = u64; @@ -11,8 +16,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = Py_ssize_t::MIN; +pub const PY_SSIZE_T_MAX: Py_ssize_t = Py_ssize_t::MAX; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index a6caf421ff6..cc16e554ca9 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -9,8 +9,8 @@ use std::os::raw::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; -opaque_struct!(PyThreadState); -opaque_struct!(PyInterpreterState); +opaque_struct!(pub PyThreadState); +opaque_struct!(pub PyInterpreterState); extern "C" { #[cfg(not(PyPy))] @@ -80,17 +80,14 @@ pub enum PyGILState_STATE { PyGILState_UNLOCKED, } +#[cfg(not(Py_3_14))] struct HangThread; +#[cfg(not(Py_3_14))] impl Drop for HangThread { fn drop(&mut self) { loop { - #[cfg(target_family = "unix")] - unsafe { - libc::pause(); - } - #[cfg(not(target_family = "unix"))] - std::thread::sleep(std::time::Duration::from_secs(9_999_999)); + std::thread::park(); // Block forever. } } } diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index e7ea2d2efd0..80209b5875a 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -49,12 +49,12 @@ pub const PYOS_STACK_MARGIN: c_int = 2048; // skipped PyOS_CheckStack under Microsoft C #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] -opaque_struct!(_mod); +opaque_struct!(pub _mod); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(symtable); +opaque_struct!(pub symtable); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(_node); +opaque_struct!(pub _node); #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] diff --git a/pyo3-ffi/src/refcount.rs b/pyo3-ffi/src/refcount.rs new file mode 100644 index 00000000000..fcb5f45be6a --- /dev/null +++ b/pyo3-ffi/src/refcount.rs @@ -0,0 +1,369 @@ +use crate::pyport::Py_ssize_t; +use crate::PyObject; +#[cfg(py_sys_config = "Py_REF_DEBUG")] +use std::os::raw::c_char; +#[cfg(Py_3_12)] +use std::os::raw::c_int; +#[cfg(all(Py_3_14, any(not(Py_GIL_DISABLED), target_pointer_width = "32")))] +use std::os::raw::c_long; +#[cfg(any(Py_GIL_DISABLED, all(Py_3_12, not(Py_3_14))))] +use std::os::raw::c_uint; +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +use std::os::raw::c_ulong; +use std::ptr; +#[cfg(Py_GIL_DISABLED)] +use std::sync::atomic::Ordering::Relaxed; + +#[cfg(Py_3_14)] +const _Py_STATICALLY_ALLOCATED_FLAG: c_int = 1 << 7; + +#[cfg(all(Py_3_12, not(Py_3_14)))] +const _Py_IMMORTAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + c_uint::MAX as Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (c_uint::MAX >> 2) as Py_ssize_t + } +}; + +// comments in Python.h about the choices for these constants + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + ((3 as c_ulong) << (30 as c_ulong)) as Py_ssize_t + } else { + ((5 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + _Py_IMMORTAL_INITIAL_REFCNT + | ((_Py_STATICALLY_ALLOCATED_FLAG as Py_ssize_t) << (32 as Py_ssize_t)) + } else { + ((7 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = ((1 as c_long) << (30 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_STATIC_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = + ((6 as c_long) << (28 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, Py_GIL_DISABLED))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = c_uint::MAX as Py_ssize_t; + +#[cfg(Py_GIL_DISABLED)] +pub(crate) const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; + +#[cfg(Py_GIL_DISABLED)] +const _Py_REF_SHARED_SHIFT: isize = 2; +// skipped private _Py_REF_SHARED_FLAG_MASK + +// skipped private _Py_REF_SHARED_INIT +// skipped private _Py_REF_MAYBE_WEAKREF +// skipped private _Py_REF_QUEUED +// skipped private _Py_REF_MERGED + +// skipped private _Py_REF_SHARED + +extern "C" { + #[cfg(all(Py_3_14, Py_LIMITED_API))] + pub fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t; +} + +#[cfg(not(all(Py_3_14, Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + #[cfg(Py_GIL_DISABLED)] + { + let local = (*ob).ob_ref_local.load(Relaxed); + if local == _Py_IMMORTAL_REFCNT_LOCAL { + #[cfg(not(Py_3_14))] + return _Py_IMMORTAL_REFCNT; + #[cfg(Py_3_14)] + return _Py_IMMORTAL_INITIAL_REFCNT; + } + let shared = (*ob).ob_ref_shared.load(Relaxed); + local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) + } + + #[cfg(all(Py_LIMITED_API, Py_3_14))] + { + Py_REFCNT(ob) + } + + #[cfg(all(not(Py_GIL_DISABLED), not(all(Py_LIMITED_API, Py_3_14)), Py_3_12))] + { + (*ob).ob_refcnt.ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] + { + (*ob).ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] + { + _Py_REFCNT(ob) + } +} + +#[cfg(Py_3_12)] +#[inline(always)] +unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + #[cfg(all(target_pointer_width = "64", not(Py_GIL_DISABLED)))] + { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int + } + + #[cfg(all(target_pointer_width = "32", not(Py_GIL_DISABLED)))] + { + #[cfg(not(Py_3_14))] + { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int + } + + #[cfg(Py_3_14)] + { + ((*op).ob_refcnt.ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT) as c_int + } + } + + #[cfg(Py_GIL_DISABLED)] + { + ((*op).ob_ref_local.load(Relaxed) == _Py_IMMORTAL_REFCNT_LOCAL) as c_int + } +} + +// skipped _Py_IsStaticImmortal + +// TODO: Py_SET_REFCNT + +extern "C" { + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_INCREF_IncRefTotal(); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_DECREF_DecRefTotal(); + + #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] + fn _Py_Dealloc(arg1: *mut PyObject); + + #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] + pub fn Py_IncRef(o: *mut PyObject); + #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] + pub fn Py_DecRef(o: *mut PyObject); + + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_IncRef(o: *mut PyObject); + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_DecRef(o: *mut PyObject); + + #[cfg(GraalPy)] + fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; +} + +#[inline(always)] +pub unsafe fn Py_INCREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + ))] + { + // _Py_IncRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_IncRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_IncRef(op); + } + } + + // version-specific builds are allowed to directly manipulate the reference count + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + )))] + { + #[cfg(all(Py_3_14, target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt; + if (cur_refcnt as i32) < 0 { + return; + } + (*op).ob_refcnt.ob_refcnt = cur_refcnt.wrapping_add(1); + } + + #[cfg(all(Py_3_12, not(Py_3_14), target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; + let new_refcnt = cur_refcnt.wrapping_add(1); + if new_refcnt == 0 { + return; + } + (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; + } + + #[cfg(all(Py_3_12, target_pointer_width = "32"))] + { + if _Py_IsImmortal(op) != 0 { + return; + } + (*op).ob_refcnt.ob_refcnt += 1 + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt += 1 + } + + // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + } +} + +// skipped _Py_DecRefShared +// skipped _Py_DecRefSharedDebug +// skipped _Py_MergeZeroLocalRefcount + +#[inline(always)] +#[cfg_attr( + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), + track_caller +)] +pub unsafe fn Py_DECREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + ))] + { + // _Py_DecRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_DecRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_DecRef(op); + } + } + + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + )))] + { + #[cfg(Py_3_12)] + if _Py_IsImmortal(op) != 0 { + return; + } + + // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + _Py_DECREF_DecRefTotal(); + + #[cfg(Py_3_12)] + { + (*op).ob_refcnt.ob_refcnt -= 1; + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + if (*op).ob_refcnt.ob_refcnt < 0 { + let location = std::panic::Location::caller(); + let filename = std::ffi::CString::new(location.file()).unwrap(); + _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); + } + + if (*op).ob_refcnt.ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt -= 1; + + if (*op).ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + } +} + +#[inline] +pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { + let tmp = *op; + if !tmp.is_null() { + *op = ptr::null_mut(); + Py_DECREF(tmp); + } +} + +#[inline] +pub unsafe fn Py_XINCREF(op: *mut PyObject) { + if !op.is_null() { + Py_INCREF(op) + } +} + +#[inline] +pub unsafe fn Py_XDECREF(op: *mut PyObject) { + if !op.is_null() { + Py_DECREF(op) + } +} + +extern "C" { + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; +} + +// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here +// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { + Py_INCREF(obj); + obj +} + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { + Py_XINCREF(obj); + obj +} diff --git a/pyo3-ffi/src/setobject.rs b/pyo3-ffi/src/setobject.rs index 9d5351fc798..87e33e803f4 100644 --- a/pyo3-ffi/src/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -39,11 +39,7 @@ pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { (*so).used } -#[cfg(not(Py_LIMITED_API))] -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PySet_Dummy: *mut PyObject; -} +// skipped _PySet_Dummy extern "C" { #[cfg(not(Py_LIMITED_API))] diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index 305dc290fa8..88a1bf90314 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -4,16 +4,18 @@ use std::os::raw::c_int; use std::ptr::addr_of_mut; #[cfg(all(not(PyPy), Py_LIMITED_API, not(GraalPy)))] -opaque_struct!(PyWeakReference); +opaque_struct!(pub PyWeakReference); #[cfg(all(not(PyPy), not(Py_LIMITED_API), not(GraalPy)))] pub use crate::_PyWeakReference as PyWeakReference; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + // TODO: PyO3 is depending on this symbol in `reference.rs`, we should change this and + // remove the export as this is a private symbol. pub static mut _PyWeakref_RefType: PyTypeObject; - pub static mut _PyWeakref_ProxyType: PyTypeObject; - pub static mut _PyWeakref_CallableProxyType: PyTypeObject; + static mut _PyWeakref_ProxyType: PyTypeObject; + static mut _PyWeakref_CallableProxyType: PyTypeObject; #[cfg(PyPy)] #[link_name = "PyPyWeakref_CheckRef"] diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml index d861ec7b6d5..98dfdebbd10 100644 --- a/pyo3-introspection/Cargo.toml +++ b/pyo3-introspection/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-introspection" -version = "0.25.0-dev" +version = "0.25.0" description = "Introspect dynamic libraries built with PyO3 to get metadata about the exported Python types" authors = ["PyO3 Project and Contributors "] homepage = "https://github.com/pyo3/pyo3" diff --git a/pyo3-introspection/src/introspection.rs b/pyo3-introspection/src/introspection.rs index 15fd9d92b84..665109d0a66 100644 --- a/pyo3-introspection/src/introspection.rs +++ b/pyo3-introspection/src/introspection.rs @@ -1,4 +1,4 @@ -use crate::model::{Class, Function, Module}; +use crate::model::{Argument, Arguments, Class, Function, Module, VariableLengthArgument}; use anyhow::{bail, ensure, Context, Result}; use goblin::elf::Elf; use goblin::mach::load_command::CommandVariant; @@ -7,6 +7,7 @@ use goblin::mach::{Mach, MachO, SingleArch}; use goblin::pe::PE; use goblin::Object; use serde::Deserialize; +use std::cmp::Ordering; use std::collections::HashMap; use std::fs; use std::path::Path; @@ -21,19 +22,23 @@ pub fn introspect_cdylib(library_path: impl AsRef, main_module_name: &str) /// Parses the introspection chunks found in the binary fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result { - let chunks_by_id = chunks - .iter() - .map(|c| { - ( - match c { - Chunk::Module { id, .. } => id, - Chunk::Class { id, .. } => id, - Chunk::Function { id, .. } => id, - }, - c, - ) - }) - .collect::>(); + let mut chunks_by_id = HashMap::<&str, &Chunk>::new(); + let mut chunks_by_parent = HashMap::<&str, Vec<&Chunk>>::new(); + for chunk in chunks { + if let Some(id) = match chunk { + Chunk::Module { id, .. } => Some(id), + Chunk::Class { id, .. } => Some(id), + Chunk::Function { id, .. } => id.as_ref(), + } { + chunks_by_id.insert(id, chunk); + } + if let Some(parent) = match chunk { + Chunk::Module { .. } | Chunk::Class { .. } => None, + Chunk::Function { parent, .. } => parent.as_ref(), + } { + chunks_by_parent.entry(parent).or_default().push(chunk); + } + } // We look for the root chunk for chunk in chunks { if let Chunk::Module { @@ -43,44 +48,149 @@ fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result { } = chunk { if name == main_module_name { - return parse_module(name, members, &chunks_by_id); + return convert_module(name, members, &chunks_by_id, &chunks_by_parent); } } } bail!("No module named {main_module_name} found") } -fn parse_module( +fn convert_module( name: &str, members: &[String], - chunks_by_id: &HashMap<&String, &Chunk>, + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, ) -> Result { + let (modules, classes, functions) = convert_members( + &members + .iter() + .filter_map(|id| chunks_by_id.get(id.as_str()).copied()) + .collect::>(), + chunks_by_id, + chunks_by_parent, + )?; + Ok(Module { + name: name.into(), + modules, + classes, + functions, + }) +} + +/// Convert a list of members of a module or a class +fn convert_members( + chunks: &[&Chunk], + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, +) -> Result<(Vec, Vec, Vec)> { let mut modules = Vec::new(); let mut classes = Vec::new(); let mut functions = Vec::new(); - for member in members { - if let Some(chunk) = chunks_by_id.get(member) { - match chunk { - Chunk::Module { + for chunk in chunks { + match chunk { + Chunk::Module { + name, + members, + id: _, + } => { + modules.push(convert_module( name, members, - id: _, - } => { - modules.push(parse_module(name, members, chunks_by_id)?); - } - Chunk::Class { name, id: _ } => classes.push(Class { name: name.into() }), - Chunk::Function { name, id: _ } => functions.push(Function { name: name.into() }), + chunks_by_id, + chunks_by_parent, + )?); } + Chunk::Class { name, id } => { + classes.push(convert_class(id, name, chunks_by_id, chunks_by_parent)?) + } + Chunk::Function { + name, + id: _, + arguments, + parent: _, + decorators, + } => functions.push(convert_function(name, arguments, decorators)), } } - Ok(Module { + Ok((modules, classes, functions)) +} + +fn convert_class( + id: &str, + name: &str, + chunks_by_id: &HashMap<&str, &Chunk>, + chunks_by_parent: &HashMap<&str, Vec<&Chunk>>, +) -> Result { + let (nested_modules, nested_classes, mut methods) = convert_members( + chunks_by_parent + .get(&id) + .map(Vec::as_slice) + .unwrap_or_default(), + chunks_by_id, + chunks_by_parent, + )?; + ensure!( + nested_modules.is_empty(), + "Classes cannot contain nested modules" + ); + ensure!( + nested_classes.is_empty(), + "Nested classes are not supported yet" + ); + // We sort methods to get a stable output + methods.sort_by(|l, r| match l.name.cmp(&r.name) { + Ordering::Equal => { + // We put the getter before the setter + if l.decorators.iter().any(|d| d == "property") { + Ordering::Less + } else if r.decorators.iter().any(|d| d == "property") { + Ordering::Greater + } else { + // We pick an ordering based on decorators + l.decorators.cmp(&r.decorators) + } + } + o => o, + }); + Ok(Class { name: name.into(), - modules, - classes, - functions, + methods, }) } +fn convert_function(name: &str, arguments: &ChunkArguments, decorators: &[String]) -> Function { + Function { + name: name.into(), + decorators: decorators.to_vec(), + arguments: Arguments { + positional_only_arguments: arguments.posonlyargs.iter().map(convert_argument).collect(), + arguments: arguments.args.iter().map(convert_argument).collect(), + vararg: arguments + .vararg + .as_ref() + .map(convert_variable_length_argument), + keyword_only_arguments: arguments.kwonlyargs.iter().map(convert_argument).collect(), + kwarg: arguments + .kwarg + .as_ref() + .map(convert_variable_length_argument), + }, + } +} + +fn convert_argument(arg: &ChunkArgument) -> Argument { + Argument { + name: arg.name.clone(), + default_value: arg.default.clone(), + } +} + +fn convert_variable_length_argument(arg: &ChunkArgument) -> VariableLengthArgument { + VariableLengthArgument { + name: arg.name.clone(), + } +} + fn find_introspection_chunks_in_binary_object(path: &Path) -> Result> { let library_content = fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; @@ -250,7 +360,34 @@ enum Chunk { name: String, }, Function { - id: String, + #[serde(default)] + id: Option, name: String, + arguments: ChunkArguments, + #[serde(default)] + parent: Option, + #[serde(default)] + decorators: Vec, }, } + +#[derive(Deserialize)] +struct ChunkArguments { + #[serde(default)] + posonlyargs: Vec, + #[serde(default)] + args: Vec, + #[serde(default)] + vararg: Option, + #[serde(default)] + kwonlyargs: Vec, + #[serde(default)] + kwarg: Option, +} + +#[derive(Deserialize)] +struct ChunkArgument { + name: String, + #[serde(default)] + default: Option, +} diff --git a/pyo3-introspection/src/model.rs b/pyo3-introspection/src/model.rs index 73a4c27d082..021475b9392 100644 --- a/pyo3-introspection/src/model.rs +++ b/pyo3-introspection/src/model.rs @@ -9,9 +9,40 @@ pub struct Module { #[derive(Debug, Eq, PartialEq, Clone, Hash)] pub struct Class { pub name: String, + pub methods: Vec, } #[derive(Debug, Eq, PartialEq, Clone, Hash)] pub struct Function { pub name: String, + /// decorator like 'property' or 'staticmethod' + pub decorators: Vec, + pub arguments: Arguments, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Arguments { + /// Arguments before / + pub positional_only_arguments: Vec, + /// Regular arguments (between / and *) + pub arguments: Vec, + /// *vararg + pub vararg: Option, + /// Arguments after * + pub keyword_only_arguments: Vec, + /// **kwarg + pub kwarg: Option, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Argument { + pub name: String, + /// Default value as a Python expression + pub default_value: Option, +} + +/// A variable length argument ie. *vararg or **kwarg +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct VariableLengthArgument { + pub name: String, } diff --git a/pyo3-introspection/src/stubs.rs b/pyo3-introspection/src/stubs.rs index 0705911032f..cc0a11ebd38 100644 --- a/pyo3-introspection/src/stubs.rs +++ b/pyo3-introspection/src/stubs.rs @@ -1,4 +1,4 @@ -use crate::model::{Class, Function, Module}; +use crate::model::{Argument, Class, Function, Module, VariableLengthArgument}; use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -39,14 +39,153 @@ fn module_stubs(module: &Module) -> String { for function in &module.functions { elements.push(function_stubs(function)); } - elements.push(String::new()); // last line jump - elements.join("\n") + + // We insert two line jumps (i.e. empty strings) only above and below multiple line elements (classes with methods, functions with decorators) + let mut output = String::new(); + for element in elements { + let is_multiline = element.contains('\n'); + if is_multiline && !output.is_empty() && !output.ends_with("\n\n") { + output.push('\n'); + } + output.push_str(&element); + output.push('\n'); + if is_multiline { + output.push('\n'); + } + } + // We remove a line jump at the end if they are two + if output.ends_with("\n\n") { + output.pop(); + } + output } fn class_stubs(class: &Class) -> String { - format!("class {}: ...", class.name) + let mut buffer = format!("class {}:", class.name); + if class.methods.is_empty() { + buffer.push_str(" ..."); + return buffer; + } + for method in &class.methods { + // We do the indentation + buffer.push_str("\n "); + buffer.push_str(&function_stubs(method).replace('\n', "\n ")); + } + buffer } fn function_stubs(function: &Function) -> String { - format!("def {}(*args, **kwargs): ...", function.name) + // Signature + let mut parameters = Vec::new(); + for argument in &function.arguments.positional_only_arguments { + parameters.push(argument_stub(argument)); + } + if !function.arguments.positional_only_arguments.is_empty() { + parameters.push("/".into()); + } + for argument in &function.arguments.arguments { + parameters.push(argument_stub(argument)); + } + if let Some(argument) = &function.arguments.vararg { + parameters.push(format!("*{}", variable_length_argument_stub(argument))); + } else if !function.arguments.keyword_only_arguments.is_empty() { + parameters.push("*".into()); + } + for argument in &function.arguments.keyword_only_arguments { + parameters.push(argument_stub(argument)); + } + if let Some(argument) = &function.arguments.kwarg { + parameters.push(format!("**{}", variable_length_argument_stub(argument))); + } + let output = format!("def {}({}): ...", function.name, parameters.join(", ")); + if function.decorators.is_empty() { + return output; + } + let mut buffer = String::new(); + for decorator in &function.decorators { + buffer.push('@'); + buffer.push_str(decorator); + buffer.push('\n'); + } + buffer.push_str(&output); + buffer +} + +fn argument_stub(argument: &Argument) -> String { + let mut output = argument.name.clone(); + if let Some(default_value) = &argument.default_value { + output.push('='); + output.push_str(default_value); + } + output +} + +fn variable_length_argument_stub(argument: &VariableLengthArgument) -> String { + argument.name.clone() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Arguments; + + #[test] + fn function_stubs_with_variable_length() { + let function = Function { + name: "func".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: None, + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: None, + }], + vararg: Some(VariableLengthArgument { + name: "varargs".into(), + }), + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: None, + }], + kwarg: Some(VariableLengthArgument { + name: "kwarg".into(), + }), + }, + }; + assert_eq!( + "def func(posonly, /, arg, *varargs, karg, **kwarg): ...", + function_stubs(&function) + ) + } + + #[test] + fn function_stubs_without_variable_length() { + let function = Function { + name: "afunc".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: Some("1".into()), + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: Some("True".into()), + }], + vararg: None, + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: Some("\"foo\"".into()), + }], + kwarg: None, + }, + }; + assert_eq!( + "def afunc(posonly=1, /, arg=True, *, karg=\"foo\"): ...", + function_stubs(&function) + ) + } } diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs index 37070a53a13..302efd1ea47 100644 --- a/pyo3-introspection/tests/test.rs +++ b/pyo3-introspection/tests/test.rs @@ -42,9 +42,10 @@ fn pytests_stubs() -> Result<()> { file_name.display() ) }); + // We normalize line jumps for compatibility with Windows assert_eq!( - &expected_file_content.replace('\r', ""), // Windows compatibility - actual_file_content, + expected_file_content.replace('\r', ""), + actual_file_content.replace('\r', ""), "The content of file {} is different", file_name.display() ) diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 5730df1ae1d..d626fa1ebf7 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.25.0-dev" +version = "0.25.0" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -17,7 +17,7 @@ rust-version = "1.63" [dependencies] heck = "0.5" proc-macro2 = { version = "1.0.60", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] @@ -26,7 +26,7 @@ default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0-dev" } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0" } [lints] workspace = true diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs index 19a12801065..ac3894d6419 100644 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -27,6 +27,7 @@ pub mod kw { syn::custom_keyword!(hash); syn::custom_keyword!(into_py_with); syn::custom_keyword!(item); + syn::custom_keyword!(immutable_type); syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); @@ -45,6 +46,7 @@ pub mod kw { syn::custom_keyword!(transparent); syn::custom_keyword!(unsendable); syn::custom_keyword!(weakref); + syn::custom_keyword!(generic); syn::custom_keyword!(gil_used); } @@ -350,37 +352,7 @@ impl ToTokens for OptionalKeywordAttribute { } } -#[derive(Debug, Clone)] -pub struct ExprPathWrap { - pub from_lit_str: bool, - pub expr_path: ExprPath, -} - -impl Parse for ExprPathWrap { - fn parse(input: ParseStream<'_>) -> Result { - match input.parse::() { - Ok(expr_path) => Ok(ExprPathWrap { - from_lit_str: false, - expr_path, - }), - Err(e) => match input.parse::>() { - Ok(LitStrValue(expr_path)) => Ok(ExprPathWrap { - from_lit_str: true, - expr_path, - }), - Err(_) => Err(e), - }, - } - } -} - -impl ToTokens for ExprPathWrap { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.expr_path.to_tokens(tokens) - } -} - -pub type FromPyWithAttribute = KeywordAttribute; +pub type FromPyWithAttribute = KeywordAttribute; pub type IntoPyWithAttribute = KeywordAttribute; pub type DefaultAttribute = OptionalKeywordAttribute; diff --git a/pyo3-macros-backend/src/derive_attributes.rs b/pyo3-macros-backend/src/derive_attributes.rs new file mode 100644 index 00000000000..63328a107b4 --- /dev/null +++ b/pyo3-macros-backend/src/derive_attributes.rs @@ -0,0 +1,217 @@ +use crate::attributes::{ + self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, + IntoPyWithAttribute, RenameAllAttribute, +}; +use proc_macro2::Span; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned; +use syn::{parenthesized, Attribute, LitStr, Result, Token}; + +/// Attributes for deriving `FromPyObject`/`IntoPyObject` scoped on containers. +pub enum ContainerAttribute { + /// Treat the Container as a Wrapper, operate directly on its field + Transparent(attributes::kw::transparent), + /// Force every field to be extracted from item of source Python object. + ItemAll(attributes::kw::from_item_all), + /// Change the name of an enum variant in the generated error message. + ErrorAnnotation(LitStr), + /// Change the path for the pyo3 crate + Crate(CrateAttribute), + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + RenameAll(RenameAllAttribute), +} + +impl Parse for ContainerAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::transparent) { + let kw: attributes::kw::transparent = input.parse()?; + Ok(ContainerAttribute::Transparent(kw)) + } else if lookahead.peek(attributes::kw::from_item_all) { + let kw: attributes::kw::from_item_all = input.parse()?; + Ok(ContainerAttribute::ItemAll(kw)) + } else if lookahead.peek(attributes::kw::annotation) { + let _: attributes::kw::annotation = input.parse()?; + let _: Token![=] = input.parse()?; + input.parse().map(ContainerAttribute::ErrorAnnotation) + } else if lookahead.peek(Token![crate]) { + input.parse().map(ContainerAttribute::Crate) + } else if lookahead.peek(attributes::kw::rename_all) { + input.parse().map(ContainerAttribute::RenameAll) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Default)] +pub struct ContainerAttributes { + /// Treat the Container as a Wrapper, operate directly on its field + pub transparent: Option, + /// Force every field to be extracted from item of source Python object. + pub from_item_all: Option, + /// Change the name of an enum variant in the generated error message. + pub annotation: Option, + /// Change the path for the pyo3 crate + pub krate: Option, + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + pub rename_all: Option, +} + +impl ContainerAttributes { + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = ContainerAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: ContainerAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + ContainerAttribute::Transparent(transparent) => set_option!(transparent), + ContainerAttribute::ItemAll(from_item_all) => set_option!(from_item_all), + ContainerAttribute::ErrorAnnotation(annotation) => set_option!(annotation), + ContainerAttribute::Crate(krate) => set_option!(krate), + ContainerAttribute::RenameAll(rename_all) => set_option!(rename_all), + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub enum FieldGetter { + GetItem(attributes::kw::item, Option), + GetAttr(attributes::kw::attribute, Option), +} + +impl FieldGetter { + pub fn span(&self) -> Span { + match self { + FieldGetter::GetItem(item, _) => item.span, + FieldGetter::GetAttr(attribute, _) => attribute.span, + } + } +} + +pub enum FieldAttribute { + Getter(FieldGetter), + FromPyWith(FromPyWithAttribute), + IntoPyWith(IntoPyWithAttribute), + Default(DefaultAttribute), +} + +impl Parse for FieldAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::attribute) { + let attr_kw: attributes::kw::attribute = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let attr_name: LitStr = content.parse()?; + if !content.is_empty() { + return Err(content.error( + "expected at most one argument: `attribute` or `attribute(\"name\")`", + )); + } + ensure_spanned!( + !attr_name.value().is_empty(), + attr_name.span() => "attribute name cannot be empty" + ); + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, Some(attr_name)))) + } else { + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, None))) + } + } else if lookahead.peek(attributes::kw::item) { + let item_kw: attributes::kw::item = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let key = content.parse()?; + if !content.is_empty() { + return Err( + content.error("expected at most one argument: `item` or `item(key)`") + ); + } + Ok(Self::Getter(FieldGetter::GetItem(item_kw, Some(key)))) + } else { + Ok(Self::Getter(FieldGetter::GetItem(item_kw, None))) + } + } else if lookahead.peek(attributes::kw::from_py_with) { + input.parse().map(Self::FromPyWith) + } else if lookahead.peek(attributes::kw::into_py_with) { + input.parse().map(FieldAttribute::IntoPyWith) + } else if lookahead.peek(Token![default]) { + input.parse().map(Self::Default) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct FieldAttributes { + pub getter: Option, + pub from_py_with: Option, + pub into_py_with: Option, + pub default: Option, +} + +impl FieldAttributes { + /// Extract the field attributes. + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = FieldAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + set_option!($key, concat!("`", stringify!($key), "` may only be specified once")) + }; + ($key:ident, $msg: expr) => {{ + ensure_spanned!( + self.$key.is_none(), + $key.span() => $msg + ); + self.$key = Some($key); + }} + } + + match option { + FieldAttribute::Getter(getter) => { + set_option!(getter, "only one of `attribute` or `item` can be provided") + } + FieldAttribute::FromPyWith(from_py_with) => set_option!(from_py_with), + FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), + FieldAttribute::Default(default) => set_option!(default), + } + Ok(()) + } +} diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 68f95e794a8..2c288709f93 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,18 +1,11 @@ -use crate::attributes::{ - self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, - RenameAllAttribute, RenamingRule, -}; -use crate::utils::{self, deprecated_from_py_with, Ctx}; +use crate::attributes::{DefaultAttribute, FromPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes, FieldGetter}; +use crate::utils::{self, Ctx}; use proc_macro2::TokenStream; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ - ext::IdentExt, - parenthesized, - parse::{Parse, ParseStream}, - parse_quote, - punctuated::Punctuated, - spanned::Spanned, - Attribute, DataEnum, DeriveInput, Fields, Ident, LitStr, Result, Token, + ext::IdentExt, parse_quote, punctuated::Punctuated, spanned::Spanned, DataEnum, DeriveInput, + Fields, Ident, Result, Token, }; /// Describes derivation input of an enum. @@ -26,7 +19,11 @@ impl<'a> Enum<'a> { /// /// `data_enum` is the `syn` representation of the input enum, `ident` is the /// `Identifier` of the enum. - fn new(data_enum: &'a DataEnum, ident: &'a Ident, options: ContainerOptions) -> Result { + fn new( + data_enum: &'a DataEnum, + ident: &'a Ident, + options: ContainerAttributes, + ) -> Result { ensure_spanned!( !data_enum.variants.is_empty(), ident.span() => "cannot derive FromPyObject for empty enum" @@ -35,7 +32,7 @@ impl<'a> Enum<'a> { .variants .iter() .map(|variant| { - let mut variant_options = ContainerOptions::from_attrs(&variant.attrs)?; + let mut variant_options = ContainerAttributes::from_attrs(&variant.attrs)?; if let Some(rename_all) = &options.rename_all { ensure_spanned!( variant_options.rename_all.is_none(), @@ -149,7 +146,7 @@ impl<'a> Container<'a> { /// Construct a container based on fields, identifier and attributes. /// /// Fails if the variant has no fields or incompatible attributes. - fn new(fields: &'a Fields, path: syn::Path, options: ContainerOptions) -> Result { + fn new(fields: &'a Fields, path: syn::Path, options: ContainerAttributes) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { ensure_spanned!( @@ -160,7 +157,7 @@ impl<'a> Container<'a> { .unnamed .iter() .map(|field| { - let attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( attrs.getter.is_none(), field.span() => "`getter` is not permitted on tuple struct elements." @@ -180,7 +177,7 @@ impl<'a> Container<'a> { // explicit annotation. let field = tuple_fields.pop().unwrap(); ContainerType::TupleNewtype(field.from_py_with) - } else if options.transparent { + } else if options.transparent.is_some() { bail_spanned!( fields.span() => "transparent structs and variants can only have 1 field" ); @@ -197,17 +194,17 @@ impl<'a> Container<'a> { .ident .as_ref() .expect("Named fields should have identifiers"); - let mut attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let mut attrs = FieldAttributes::from_attrs(&field.attrs)?; if let Some(ref from_item_all) = options.from_item_all { - if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(None)) + if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(parse_quote!(item), None)) { match replaced { - FieldGetter::GetItem(Some(item_name)) => { - attrs.getter = Some(FieldGetter::GetItem(Some(item_name))); + FieldGetter::GetItem(item, Some(item_name)) => { + attrs.getter = Some(FieldGetter::GetItem(item, Some(item_name))); } - FieldGetter::GetItem(None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), - FieldGetter::GetAttr(_) => bail_spanned!( + FieldGetter::GetItem(_, None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), + FieldGetter::GetAttr(_, _) => bail_spanned!( from_item_all.span() => "The struct is already annotated with `from_item_all`, `attribute` is not allowed" ), } @@ -226,7 +223,7 @@ impl<'a> Container<'a> { bail_spanned!( fields.span() => "cannot derive FromPyObject for structs and variants with only default values" ) - } else if options.transparent { + } else if options.transparent.is_some() { ensure_spanned!( struct_fields.len() == 1, fields.span() => "transparent structs and variants can only have 1 field" @@ -304,13 +301,10 @@ impl<'a> Container<'a> { value: expr_path, }) = from_py_with { - let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); - let extractor = quote_spanned! { kw.span => { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } }; quote! { - #deprecation Ok(#self_ty { #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, obj, #struct_name, #field_name)? }) @@ -327,13 +321,10 @@ impl<'a> Container<'a> { value: expr_path, }) = from_py_with { - let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); - let extractor = quote_spanned! { kw.span => { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } }; quote! { - #deprecation #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, obj, #struct_name, 0).map(#self_ty) } } else { @@ -367,14 +358,7 @@ impl<'a> Container<'a> { }} }); - let deprecations = struct_fields - .iter() - .filter_map(|fields| fields.from_py_with.as_ref()) - .filter_map(|kw| deprecated_from_py_with(&kw.value)) - .collect::(); - quote!( - #deprecations match #pyo3_path::types::PyAnyMethods::extract(obj) { ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), ::std::result::Result::Err(err) => ::std::result::Result::Err(err), @@ -390,24 +374,28 @@ impl<'a> Container<'a> { for field in struct_fields { let ident = field.ident; let field_name = ident.unraw().to_string(); - let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { - FieldGetter::GetAttr(Some(name)) => { + let getter = match field + .getter + .as_ref() + .unwrap_or(&FieldGetter::GetAttr(parse_quote!(attribute), None)) + { + FieldGetter::GetAttr(_, Some(name)) => { quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetAttr(None) => { + FieldGetter::GetAttr(_, None) => { let name = self .rename_rule .map(|rule| utils::apply_renaming_rule(rule, &field_name)); let name = name.as_deref().unwrap_or(&field_name); quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { + FieldGetter::GetItem(_, Some(syn::Lit::Str(key))) => { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #key))) } - FieldGetter::GetItem(Some(key)) => { + FieldGetter::GetItem(_, Some(key)) => { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #key)) } - FieldGetter::GetItem(None) => { + FieldGetter::GetItem(_, None) => { let name = self .rename_rule .map(|rule| utils::apply_renaming_rule(rule, &field_name)); @@ -448,229 +436,7 @@ impl<'a> Container<'a> { fields.push(quote!(#ident: #extracted)); } - let d = struct_fields - .iter() - .filter_map(|field| field.from_py_with.as_ref()) - .filter_map(|kw| deprecated_from_py_with(&kw.value)) - .collect::(); - - quote!(#d ::std::result::Result::Ok(#self_ty{#fields})) - } -} - -#[derive(Default)] -struct ContainerOptions { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - transparent: bool, - /// Force every field to be extracted from item of source Python object. - from_item_all: Option, - /// Change the name of an enum variant in the generated error message. - annotation: Option, - /// Change the path for the pyo3 crate - krate: Option, - /// Converts the field idents according to the [RenamingRule] before extraction - rename_all: Option, -} - -/// Attributes for deriving FromPyObject scoped on containers. -enum ContainerPyO3Attribute { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - Transparent(attributes::kw::transparent), - /// Force every field to be extracted from item of source Python object. - ItemAll(attributes::kw::from_item_all), - /// Change the name of an enum variant in the generated error message. - ErrorAnnotation(LitStr), - /// Change the path for the pyo3 crate - Crate(CrateAttribute), - /// Converts the field idents according to the [RenamingRule] before extraction - RenameAll(RenameAllAttribute), -} - -impl Parse for ContainerPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::transparent) { - let kw: attributes::kw::transparent = input.parse()?; - Ok(ContainerPyO3Attribute::Transparent(kw)) - } else if lookahead.peek(attributes::kw::from_item_all) { - let kw: attributes::kw::from_item_all = input.parse()?; - Ok(ContainerPyO3Attribute::ItemAll(kw)) - } else if lookahead.peek(attributes::kw::annotation) { - let _: attributes::kw::annotation = input.parse()?; - let _: Token![=] = input.parse()?; - input.parse().map(ContainerPyO3Attribute::ErrorAnnotation) - } else if lookahead.peek(Token![crate]) { - input.parse().map(ContainerPyO3Attribute::Crate) - } else if lookahead.peek(attributes::kw::rename_all) { - input.parse().map(ContainerPyO3Attribute::RenameAll) - } else { - Err(lookahead.error()) - } - } -} - -impl ContainerOptions { - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = ContainerOptions::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - ContainerPyO3Attribute::Transparent(kw) => { - ensure_spanned!( - !options.transparent, - kw.span() => "`transparent` may only be provided once" - ); - options.transparent = true; - } - ContainerPyO3Attribute::ItemAll(kw) => { - ensure_spanned!( - options.from_item_all.is_none(), - kw.span() => "`from_item_all` may only be provided once" - ); - options.from_item_all = Some(kw); - } - ContainerPyO3Attribute::ErrorAnnotation(lit_str) => { - ensure_spanned!( - options.annotation.is_none(), - lit_str.span() => "`annotation` may only be provided once" - ); - options.annotation = Some(lit_str); - } - ContainerPyO3Attribute::Crate(path) => { - ensure_spanned!( - options.krate.is_none(), - path.span() => "`crate` may only be provided once" - ); - options.krate = Some(path); - } - ContainerPyO3Attribute::RenameAll(rename_all) => { - ensure_spanned!( - options.rename_all.is_none(), - rename_all.span() => "`rename_all` may only be provided once" - ); - options.rename_all = Some(rename_all); - } - } - } - } - } - Ok(options) - } -} - -/// Attributes for deriving FromPyObject scoped on fields. -#[derive(Clone, Debug)] -struct FieldPyO3Attributes { - getter: Option, - from_py_with: Option, - default: Option, -} - -#[derive(Clone, Debug)] -enum FieldGetter { - GetItem(Option), - GetAttr(Option), -} - -enum FieldPyO3Attribute { - Getter(FieldGetter), - FromPyWith(FromPyWithAttribute), - Default(DefaultAttribute), -} - -impl Parse for FieldPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::attribute) { - let _: attributes::kw::attribute = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let attr_name: LitStr = content.parse()?; - if !content.is_empty() { - return Err(content.error( - "expected at most one argument: `attribute` or `attribute(\"name\")`", - )); - } - ensure_spanned!( - !attr_name.value().is_empty(), - attr_name.span() => "attribute name cannot be empty" - ); - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(Some( - attr_name, - )))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(None))) - } - } else if lookahead.peek(attributes::kw::item) { - let _: attributes::kw::item = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let key = content.parse()?; - if !content.is_empty() { - return Err( - content.error("expected at most one argument: `item` or `item(key)`") - ); - } - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(Some(key)))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(None))) - } - } else if lookahead.peek(attributes::kw::from_py_with) { - input.parse().map(FieldPyO3Attribute::FromPyWith) - } else if lookahead.peek(Token![default]) { - input.parse().map(FieldPyO3Attribute::Default) - } else { - Err(lookahead.error()) - } - } -} - -impl FieldPyO3Attributes { - /// Extract the field attributes. - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut getter = None; - let mut from_py_with = None; - let mut default = None; - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - FieldPyO3Attribute::Getter(field_getter) => { - ensure_spanned!( - getter.is_none(), - attr.span() => "only one of `attribute` or `item` can be provided" - ); - getter = Some(field_getter); - } - FieldPyO3Attribute::FromPyWith(from_py_with_attr) => { - ensure_spanned!( - from_py_with.is_none(), - attr.span() => "`from_py_with` may only be provided once" - ); - from_py_with = Some(from_py_with_attr); - } - FieldPyO3Attribute::Default(default_attr) => { - ensure_spanned!( - default.is_none(), - attr.span() => "`default` may only be provided once" - ); - default = Some(default_attr); - } - } - } - } - } - - Ok(FieldPyO3Attributes { - getter, - from_py_with, - default, - }) + quote!(::std::result::Result::Ok(#self_ty{#fields})) } } @@ -693,7 +459,7 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Result Foo(T)` /// adds `T: FromPyObject` on the derived implementation. pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { - let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; let ctx = &Ctx::new(&options.krate, None); let Ctx { pyo3_path, .. } = &ctx; @@ -717,7 +483,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let derives = match &tokens.data { syn::Data::Enum(en) => { - if options.transparent || options.annotation.is_some() { + if options.transparent.is_some() || options.annotation.is_some() { bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \ at top level for enums"); } diff --git a/pyo3-macros-backend/src/intopyobject.rs b/pyo3-macros-backend/src/intopyobject.rs index 787d1dd6259..2532812f6e1 100644 --- a/pyo3-macros-backend/src/intopyobject.rs +++ b/pyo3-macros-backend/src/intopyobject.rs @@ -1,173 +1,13 @@ -use crate::attributes::{self, get_pyo3_options, CrateAttribute, IntoPyWithAttribute}; -use crate::utils::Ctx; +use crate::attributes::{IntoPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes}; +use crate::utils::{self, Ctx}; use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::ext::IdentExt; -use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned as _; -use syn::{ - parenthesized, parse_quote, Attribute, DataEnum, DeriveInput, Fields, Ident, Index, Result, - Token, -}; - -/// Attributes for deriving `IntoPyObject` scoped on containers. -enum ContainerPyO3Attribute { - /// Treat the Container as a Wrapper, directly convert its field into the output object. - Transparent(attributes::kw::transparent), - /// Change the path for the pyo3 crate - Crate(CrateAttribute), -} - -impl Parse for ContainerPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::transparent) { - let kw: attributes::kw::transparent = input.parse()?; - Ok(ContainerPyO3Attribute::Transparent(kw)) - } else if lookahead.peek(Token![crate]) { - input.parse().map(ContainerPyO3Attribute::Crate) - } else { - Err(lookahead.error()) - } - } -} - -#[derive(Default)] -struct ContainerOptions { - /// Treat the Container as a Wrapper, directly convert its field into the output object. - transparent: Option, - /// Change the path for the pyo3 crate - krate: Option, -} - -impl ContainerOptions { - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = ContainerOptions::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - pyo3_attrs - .into_iter() - .try_for_each(|opt| options.set_option(opt))?; - } - } - Ok(options) - } - - fn set_option(&mut self, option: ContainerPyO3Attribute) -> syn::Result<()> { - macro_rules! set_option { - ($key:ident) => { - { - ensure_spanned!( - self.$key.is_none(), - $key.span() => concat!("`", stringify!($key), "` may only be specified once") - ); - self.$key = Some($key); - } - }; - } - - match option { - ContainerPyO3Attribute::Transparent(transparent) => set_option!(transparent), - ContainerPyO3Attribute::Crate(krate) => set_option!(krate), - } - Ok(()) - } -} - -#[derive(Debug, Clone)] -struct ItemOption { - field: Option, - span: Span, -} - -impl ItemOption { - fn span(&self) -> Span { - self.span - } -} - -enum FieldAttribute { - Item(ItemOption), - IntoPyWith(IntoPyWithAttribute), -} - -impl Parse for FieldAttribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::attribute) { - let attr: attributes::kw::attribute = input.parse()?; - bail_spanned!(attr.span => "`attribute` is not supported by `IntoPyObject`"); - } else if lookahead.peek(attributes::kw::item) { - let attr: attributes::kw::item = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let key = content.parse()?; - if !content.is_empty() { - return Err( - content.error("expected at most one argument: `item` or `item(key)`") - ); - } - Ok(FieldAttribute::Item(ItemOption { - field: Some(key), - span: attr.span, - })) - } else { - Ok(FieldAttribute::Item(ItemOption { - field: None, - span: attr.span, - })) - } - } else if lookahead.peek(attributes::kw::into_py_with) { - input.parse().map(FieldAttribute::IntoPyWith) - } else { - Err(lookahead.error()) - } - } -} - -#[derive(Clone, Debug, Default)] -struct FieldAttributes { - item: Option, - into_py_with: Option, -} - -impl FieldAttributes { - /// Extract the field attributes. - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = FieldAttributes::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - pyo3_attrs - .into_iter() - .try_for_each(|opt| options.set_option(opt))?; - } - } - Ok(options) - } +use syn::{parse_quote, DataEnum, DeriveInput, Fields, Ident, Index, Result}; - fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { - macro_rules! set_option { - ($key:ident) => { - { - ensure_spanned!( - self.$key.is_none(), - $key.span() => concat!("`", stringify!($key), "` may only be specified once") - ); - self.$key = Some($key); - } - }; - } - - match option { - FieldAttribute::Item(item) => set_option!(item), - FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), - } - Ok(()) - } -} +struct ItemOption(Option); enum IntoPyObjectTypes { Transparent(syn::Type), @@ -225,6 +65,7 @@ struct Container<'a, const REF: bool> { path: syn::Path, receiver: Option, ty: ContainerType<'a>, + rename_rule: Option, } /// Construct a container based on fields, identifier and attributes. @@ -235,18 +76,22 @@ impl<'a, const REF: bool> Container<'a, REF> { receiver: Option, fields: &'a Fields, path: syn::Path, - options: ContainerOptions, + options: ContainerAttributes, ) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is useless on tuple structs and variants." + ); let mut tuple_fields = unnamed .unnamed .iter() .map(|field| { let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( - attrs.item.is_none(), - attrs.item.unwrap().span() => "`item` is not permitted on tuple struct elements." + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`item` and `attribute` are not permitted on tuple struct elements." ); Ok(TupleStructField { field, @@ -284,8 +129,12 @@ impl<'a, const REF: bool> Container<'a, REF> { let field = named.named.iter().next().unwrap(); let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( - attrs.item.is_none(), - attrs.item.unwrap().span() => "`transparent` structs may not have `item` for the inner field" + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`transparent` structs may not have `item` nor `attribute` for the inner field" + ); + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is not permitted on `transparent` structs and variants" ); ensure_spanned!( attrs.into_py_with.is_none(), @@ -307,7 +156,12 @@ impl<'a, const REF: bool> Container<'a, REF> { Ok(NamedStructField { ident, field, - item: attrs.item, + item: attrs.getter.and_then(|getter| match getter { + crate::derive_attributes::FieldGetter::GetItem(_, lit) => { + Some(ItemOption(lit)) + } + crate::derive_attributes::FieldGetter::GetAttr(_, _) => None, + }), into_py_with: attrs.into_py_with, }) }) @@ -324,6 +178,7 @@ impl<'a, const REF: bool> Container<'a, REF> { path, receiver, ty: style, + rename_rule: options.rename_all.map(|v| v.value.rule), }; Ok(v) } @@ -407,9 +262,12 @@ impl<'a, const REF: bool> Container<'a, REF> { let key = f .item .as_ref() - .and_then(|item| item.field.as_ref()) - .map(|item| item.value()) - .unwrap_or_else(|| f.ident.unraw().to_string()); + .and_then(|item| item.0.as_ref()) + .map(|item| item.into_token_stream()) + .unwrap_or_else(|| { + let name = f.ident.unraw().to_string(); + self.rename_rule.map(|rule| utils::apply_renaming_rule(rule, &name)).unwrap_or(name).into_token_stream() + }); let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { @@ -519,7 +377,7 @@ impl<'a, const REF: bool> Enum<'a, REF> { .variants .iter() .map(|variant| { - let attrs = ContainerOptions::from_attrs(&variant.attrs)?; + let attrs = ContainerAttributes::from_attrs(&variant.attrs)?; let var_ident = &variant.ident; ensure_spanned!( @@ -582,7 +440,7 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Option<&syn::LifetimePar } pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Result { - let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; let ctx = &Ctx::new(&options.krate, None); let Ctx { pyo3_path, .. } = &ctx; @@ -614,6 +472,9 @@ pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Resu if options.transparent.is_some() { bail_spanned!(tokens.span() => "`transparent` is not supported at top level for enums"); } + if let Some(rename_all) = options.rename_all { + bail_spanned!(rename_all.span() => "`rename_all` is not supported at top level for enums"); + } let en = Enum::::new(en, &tokens.ident)?; en.build(ctx) } diff --git a/pyo3-macros-backend/src/introspection.rs b/pyo3-macros-backend/src/introspection.rs index 17ae19ee8d9..303e952b814 100644 --- a/pyo3-macros-backend/src/introspection.rs +++ b/pyo3-macros-backend/src/introspection.rs @@ -8,15 +8,18 @@ //! The JSON blobs format must be synchronized with the `pyo3_introspection::introspection.rs::Chunk` //! type that is used to parse them. +use crate::method::{FnArg, RegularArg}; +use crate::pyfunction::FunctionSignature; use crate::utils::PyO3CratePath; use proc_macro2::{Span, TokenStream}; use quote::{format_ident, quote, ToTokens}; +use std::borrow::Cow; use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; use std::hash::{Hash, Hasher}; use std::mem::take; use std::sync::atomic::{AtomicUsize, Ordering}; -use syn::{Attribute, Ident}; +use syn::{Attribute, Ident, Type, TypePath}; static GLOBAL_COUNTER_FOR_UNIQUE_NAMES: AtomicUsize = AtomicUsize::new(0); @@ -26,11 +29,11 @@ pub fn module_introspection_code<'a>( members: impl IntoIterator, members_cfg_attrs: impl IntoIterator>, ) -> TokenStream { - let stub = IntrospectionNode::Map( + IntrospectionNode::Map( [ - ("type", IntrospectionNode::String("module")), + ("type", IntrospectionNode::String("module".into())), ("id", IntrospectionNode::IntrospectionId(None)), - ("name", IntrospectionNode::String(name)), + ("name", IntrospectionNode::String(name.into())), ( "members", IntrospectionNode::List( @@ -39,7 +42,9 @@ pub fn module_introspection_code<'a>( .zip(members_cfg_attrs) .filter_map(|(member, attributes)| { if attributes.is_empty() { - Some(IntrospectionNode::IntrospectionId(Some(member))) + Some(IntrospectionNode::IntrospectionId(Some(ident_to_type( + member, + )))) } else { None // TODO: properly interpret cfg attributes } @@ -50,12 +55,7 @@ pub fn module_introspection_code<'a>( ] .into(), ) - .emit(pyo3_crate_path); - let introspection_id = introspection_id_const(); - quote! { - #stub - #introspection_id - } + .emit(pyo3_crate_path) } pub fn class_introspection_code( @@ -63,44 +63,163 @@ pub fn class_introspection_code( ident: &Ident, name: &str, ) -> TokenStream { - let stub = IntrospectionNode::Map( + IntrospectionNode::Map( [ - ("type", IntrospectionNode::String("class")), - ("id", IntrospectionNode::IntrospectionId(Some(ident))), - ("name", IntrospectionNode::String(name)), + ("type", IntrospectionNode::String("class".into())), + ( + "id", + IntrospectionNode::IntrospectionId(Some(ident_to_type(ident))), + ), + ("name", IntrospectionNode::String(name.into())), ] .into(), ) - .emit(pyo3_crate_path); - let introspection_id = introspection_id_const(); - quote! { - #stub - impl #ident { - #introspection_id + .emit(pyo3_crate_path) +} + +pub fn function_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: Option<&Ident>, + name: &str, + signature: &FunctionSignature<'_>, + first_argument: Option<&'static str>, + decorators: impl IntoIterator, + parent: Option<&Type>, +) -> TokenStream { + let mut desc = HashMap::from([ + ("type", IntrospectionNode::String("function".into())), + ("name", IntrospectionNode::String(name.into())), + ( + "arguments", + arguments_introspection_data(signature, first_argument), + ), + ]); + if let Some(ident) = ident { + desc.insert( + "id", + IntrospectionNode::IntrospectionId(Some(ident_to_type(ident))), + ); + } + let decorators = decorators + .into_iter() + .map(|d| IntrospectionNode::String(d.into())) + .collect::>(); + if !decorators.is_empty() { + desc.insert("decorators", IntrospectionNode::List(decorators)); + } + if let Some(parent) = parent { + desc.insert( + "parent", + IntrospectionNode::IntrospectionId(Some(Cow::Borrowed(parent))), + ); + } + IntrospectionNode::Map(desc).emit(pyo3_crate_path) +} + +fn arguments_introspection_data<'a>( + signature: &'a FunctionSignature<'a>, + first_argument: Option<&'a str>, +) -> IntrospectionNode<'a> { + let mut argument_desc = signature.arguments.iter().filter_map(|arg| { + if let FnArg::Regular(arg) = arg { + Some(arg) + } else { + None } + }); + + let mut posonlyargs = Vec::new(); + let mut args = Vec::new(); + let mut vararg = None; + let mut kwonlyargs = Vec::new(); + let mut kwarg = None; + + if let Some(first_argument) = first_argument { + posonlyargs.push(IntrospectionNode::Map( + [("name", IntrospectionNode::String(first_argument.into()))].into(), + )); + } + + for (i, param) in signature + .python_signature + .positional_parameters + .iter() + .enumerate() + { + let arg_desc = if let Some(arg_desc) = argument_desc.next() { + arg_desc + } else { + panic!("Less arguments than in python signature"); + }; + let arg = argument_introspection_data(param, arg_desc); + if i < signature.python_signature.positional_only_parameters { + posonlyargs.push(arg); + } else { + args.push(arg) + } + } + + if let Some(param) = &signature.python_signature.varargs { + vararg = Some(IntrospectionNode::Map( + [("name", IntrospectionNode::String(param.into()))].into(), + )); } + + for (param, _) in &signature.python_signature.keyword_only_parameters { + let arg_desc = if let Some(arg_desc) = argument_desc.next() { + arg_desc + } else { + panic!("Less arguments than in python signature"); + }; + kwonlyargs.push(argument_introspection_data(param, arg_desc)); + } + + if let Some(param) = &signature.python_signature.kwargs { + kwarg = Some(IntrospectionNode::Map( + [ + ("name", IntrospectionNode::String(param.into())), + ("kind", IntrospectionNode::String("VAR_KEYWORD".into())), + ] + .into(), + )); + } + + let mut map = HashMap::new(); + if !posonlyargs.is_empty() { + map.insert("posonlyargs", IntrospectionNode::List(posonlyargs)); + } + if !args.is_empty() { + map.insert("args", IntrospectionNode::List(args)); + } + if let Some(vararg) = vararg { + map.insert("vararg", vararg); + } + if !kwonlyargs.is_empty() { + map.insert("kwonlyargs", IntrospectionNode::List(kwonlyargs)); + } + if let Some(kwarg) = kwarg { + map.insert("kwarg", kwarg); + } + IntrospectionNode::Map(map) } -pub fn function_introspection_code(pyo3_crate_path: &PyO3CratePath, name: &str) -> TokenStream { - let stub = IntrospectionNode::Map( - [ - ("type", IntrospectionNode::String("function")), - ("id", IntrospectionNode::IntrospectionId(None)), - ("name", IntrospectionNode::String(name)), - ] - .into(), - ) - .emit(pyo3_crate_path); - let introspection_id = introspection_id_const(); - quote! { - #stub - #introspection_id +fn argument_introspection_data<'a>( + name: &'a str, + desc: &'a RegularArg<'_>, +) -> IntrospectionNode<'a> { + let mut params: HashMap<_, _> = [("name", IntrospectionNode::String(name.into()))].into(); + if desc.default_value.is_some() { + params.insert( + "default", + IntrospectionNode::String(desc.default_value().into()), + ); } + IntrospectionNode::Map(params) } enum IntrospectionNode<'a> { - String(&'a str), - IntrospectionId(Option<&'a Ident>), + String(Cow<'a, str>), + IntrospectionId(Option>), Map(HashMap<&'static str, IntrospectionNode<'a>>), List(Vec>), } @@ -125,7 +244,7 @@ impl IntrospectionNode<'_> { fn add_to_serialization(self, content: &mut ConcatenationBuilder) { match self { Self::String(string) => { - content.push_str_to_escape(string); + content.push_str_to_escape(&string); } Self::IntrospectionId(ident) => { content.push_str("\""); @@ -232,7 +351,8 @@ impl ToTokens for ConcatenationBuilderElement { } } -fn introspection_id_const() -> TokenStream { +/// Generates a new unique identifier for linking introspection objects together +pub fn introspection_id_const() -> TokenStream { let id = unique_element_id().to_string(); quote! { #[doc(hidden)] @@ -248,3 +368,13 @@ fn unique_element_id() -> u64 { .hash(&mut hasher); // If there are multiple elements in the same call site hasher.finish() } + +fn ident_to_type(ident: &Ident) -> Cow<'static, Type> { + Cow::Owned( + TypePath { + path: ident.clone().into(), + qself: None, + } + .into(), + ) +} diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 4baf58907b4..8919830dda9 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -9,6 +9,7 @@ mod utils; mod attributes; +mod derive_attributes; mod frompyobject; mod intopyobject; #[cfg(feature = "experimental-inspect")] diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index bc117874a5b..0ab5135e93a 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -27,6 +27,52 @@ pub struct RegularArg<'a> { pub option_wrapped_type: Option<&'a syn::Type>, } +impl RegularArg<'_> { + pub fn default_value(&self) -> String { + if let Self { + default_value: Some(arg_default), + .. + } = self + { + match arg_default { + // literal values + syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { + syn::Lit::Str(s) => s.token().to_string(), + syn::Lit::Char(c) => c.token().to_string(), + syn::Lit::Int(i) => i.base10_digits().to_string(), + syn::Lit::Float(f) => f.base10_digits().to_string(), + syn::Lit::Bool(b) => { + if b.value() { + "True".to_string() + } else { + "False".to_string() + } + } + _ => "...".to_string(), + }, + // None + syn::Expr::Path(syn::ExprPath { qself, path, .. }) + if qself.is_none() && path.is_ident("None") => + { + "None".to_string() + } + // others, unsupported yet so defaults to `...` + _ => "...".to_string(), + } + } else if let RegularArg { + option_wrapped_type: Some(..), + .. + } = self + { + // functions without a `#[pyo3(signature = (...))]` option + // will treat trailing `Option` arguments as having a default of `None` + "None".to_string() + } else { + "...".to_string() + } + } +} + /// Pythons *args argument #[derive(Clone, Debug)] pub struct VarargsArg<'a> { @@ -177,6 +223,14 @@ impl<'a> FnArg<'a> { } } } + + pub fn default_value(&self) -> String { + if let Self::Regular(args) = self { + args.default_value() + } else { + "...".to_string() + } + } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { @@ -593,10 +647,10 @@ impl<'a> FnSpec<'a> { .fold(first.span(), |s, next| s.join(next.span()).unwrap_or(s)); let span = span.join(last.span()).unwrap_or(span); // List all the attributes in the error message - let mut msg = format!("`{}` may not be combined with", first); + let mut msg = format!("`{first}` may not be combined with"); let mut is_first = true; for attr in &*rest { - msg.push_str(&format!(" `{}`", attr)); + msg.push_str(&format!(" `{attr}`")); if is_first { is_first = false; } else { @@ -606,7 +660,7 @@ impl<'a> FnSpec<'a> { if !rest.is_empty() { msg.push_str(" and"); } - msg.push_str(&format!(" `{}`", last)); + msg.push_str(&format!(" `{last}`")); bail_spanned!(span => msg) } }; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 7ee165cbd27..860a3b6d857 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,7 +1,7 @@ //! Code generation for the function that initializes a python module and adds classes and function. #[cfg(feature = "experimental-inspect")] -use crate::introspection::module_introspection_code; +use crate::introspection::{introspection_id_const, module_introspection_code}; use crate::{ attributes::{ self, kw, take_attributes, take_pyo3_options, CrateAttribute, GILUsedAttribute, @@ -149,6 +149,8 @@ pub fn pymodule_module_impl( } let mut pymodule_init = None; + let mut module_consts = Vec::new(); + let mut module_consts_cfg_attrs = Vec::new(); for item in &mut *items { match item { @@ -168,7 +170,7 @@ pub fn pymodule_module_impl( Item::Fn(item_fn) => { ensure_spanned!( !has_attribute(&item_fn.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); let is_pymodule_init = find_and_remove_attribute(&mut item_fn.attrs, "pymodule_init"); @@ -199,7 +201,7 @@ pub fn pymodule_module_impl( Item::Struct(item_struct) => { ensure_spanned!( !has_attribute(&item_struct.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_struct.attrs, "pyclass") || has_attribute_with_namespace( @@ -227,7 +229,7 @@ pub fn pymodule_module_impl( Item::Enum(item_enum) => { ensure_spanned!( !has_attribute(&item_enum.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_enum.attrs, "pyclass") || has_attribute_with_namespace(&item_enum.attrs, Some(pyo3_path), &["pyclass"]) @@ -251,7 +253,7 @@ pub fn pymodule_module_impl( Item::Mod(item_mod) => { ensure_spanned!( !has_attribute(&item_mod.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_mod.attrs, "pymodule") || has_attribute_with_namespace(&item_mod.attrs, Some(pyo3_path), &["pymodule"]) @@ -278,61 +280,63 @@ pub fn pymodule_module_impl( Item::ForeignMod(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Trait(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Const(item) => { - ensure_spanned!( - !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" - ); + if !find_and_remove_attribute(&mut item.attrs, "pymodule_export") { + continue; + } + + module_consts.push(item.ident.clone()); + module_consts_cfg_attrs.push(get_cfg_attributes(&item.attrs)); } Item::Static(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Macro(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::ExternCrate(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Impl(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::TraitAlias(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Type(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Union(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } _ => (), @@ -348,6 +352,10 @@ pub fn pymodule_module_impl( ); #[cfg(not(feature = "experimental-inspect"))] let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; let module_def = quote! {{ use #pyo3_path::impl_::pymodule as impl_; @@ -368,6 +376,8 @@ pub fn pymodule_module_impl( options.gil_used.map_or(true, |op| op.value.value), ); + let module_consts_names = module_consts.iter().map(|i| i.unraw().to_string()); + Ok(quote!( #(#attrs)* #vis #mod_token #ident { @@ -375,6 +385,7 @@ pub fn pymodule_module_impl( #initialization #introspection + #introspection_id fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { use #pyo3_path::impl_::pymodule::PyAddToModule; @@ -382,6 +393,12 @@ pub fn pymodule_module_impl( #(#module_items_cfg_attrs)* #module_items::_PYO3_DEF.add_to_module(module)?; )* + + #( + #(#module_consts_cfg_attrs)* + #pyo3_path::types::PyModuleMethods::add(module, #module_consts_names, #module_consts)?; + )* + #pymodule_init ::std::result::Result::Ok(()) } @@ -418,6 +435,10 @@ pub fn pymodule_function_impl( let introspection = module_introspection_code(pyo3_path, &name.to_string(), &[], &[]); #[cfg(not(feature = "experimental-inspect"))] let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; // Module function called with optional Python<'_> marker as first arg, followed by the module. let mut module_args = Vec::new(); @@ -432,6 +453,7 @@ pub fn pymodule_function_impl( #vis mod #ident { #initialization #introspection + #introspection_id } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -466,7 +488,7 @@ fn module_initialization( gil_used: bool, ) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; - let pyinit_symbol = format!("PyInit_{}", name); + let pyinit_symbol = format!("PyInit_{name}"); let name = name.to_string(); let pyo3_name = LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx); diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index 6ff065e7458..087e97f500c 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,4 +1,4 @@ -use crate::utils::{deprecated_from_py_with, Ctx, TypeExt as _}; +use crate::utils::{Ctx, TypeExt as _}; use crate::{ attributes::FromPyWithAttribute, method::{FnArg, FnSpec, RegularArg}, @@ -62,9 +62,7 @@ pub fn impl_arg_params( .filter_map(|(i, arg)| { let from_py_with = &arg.from_py_with()?.value; let from_py_with_holder = format_ident!("from_py_with_{}", i); - let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); Some(quote_spanned! { from_py_with.span() => - #d let #from_py_with_holder = #from_py_with; }) }) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index ed15917279e..ab86138338b 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -15,7 +15,7 @@ use crate::attributes::{ StrFormatterAttribute, }; #[cfg(feature = "experimental-inspect")] -use crate::introspection::class_introspection_code; +use crate::introspection::{class_introspection_code, introspection_id_const}; use crate::konst::{ConstAttributes, ConstSpec}; use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyfunction::ConstructorAttribute; @@ -25,7 +25,7 @@ use crate::pymethod::{ MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, __RICHCMP__, __STR__, }; -use crate::pyversions::is_abi3_before; +use crate::pyversions::{is_abi3_before, is_py_before}; use crate::utils::{self, apply_renaming_rule, Ctx, LitCStr, PythonDoc}; use crate::PyFunctionOptions; @@ -71,6 +71,7 @@ pub struct PyClassPyO3Options { pub freelist: Option, pub frozen: Option, pub hash: Option, + pub immutable_type: Option, pub mapping: Option, pub module: Option, pub name: Option, @@ -82,6 +83,7 @@ pub struct PyClassPyO3Options { pub subclass: Option, pub unsendable: Option, pub weakref: Option, + pub generic: Option, } pub enum PyClassPyO3Option { @@ -94,6 +96,7 @@ pub enum PyClassPyO3Option { Frozen(kw::frozen), GetAll(kw::get_all), Hash(kw::hash), + ImmutableType(kw::immutable_type), Mapping(kw::mapping), Module(ModuleAttribute), Name(NameAttribute), @@ -105,6 +108,7 @@ pub enum PyClassPyO3Option { Subclass(kw::subclass), Unsendable(kw::unsendable), Weakref(kw::weakref), + Generic(kw::generic), } impl Parse for PyClassPyO3Option { @@ -128,6 +132,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::GetAll) } else if lookahead.peek(attributes::kw::hash) { input.parse().map(PyClassPyO3Option::Hash) + } else if lookahead.peek(attributes::kw::immutable_type) { + input.parse().map(PyClassPyO3Option::ImmutableType) } else if lookahead.peek(attributes::kw::mapping) { input.parse().map(PyClassPyO3Option::Mapping) } else if lookahead.peek(attributes::kw::module) { @@ -150,6 +156,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Unsendable) } else if lookahead.peek(attributes::kw::weakref) { input.parse().map(PyClassPyO3Option::Weakref) + } else if lookahead.peek(attributes::kw::generic) { + input.parse().map(PyClassPyO3Option::Generic) } else { Err(lookahead.error()) } @@ -203,6 +211,13 @@ impl PyClassPyO3Options { PyClassPyO3Option::Freelist(freelist) => set_option!(freelist), PyClassPyO3Option::Frozen(frozen) => set_option!(frozen), PyClassPyO3Option::GetAll(get_all) => set_option!(get_all), + PyClassPyO3Option::ImmutableType(immutable_type) => { + ensure_spanned!( + !(is_py_before(3, 10) || is_abi3_before(3, 14)), + immutable_type.span() => "`immutable_type` requires Python >= 3.10 or >= 3.14 (ABI3)" + ); + set_option!(immutable_type) + } PyClassPyO3Option::Hash(hash) => set_option!(hash), PyClassPyO3Option::Mapping(mapping) => set_option!(mapping), PyClassPyO3Option::Module(module) => set_option!(module), @@ -221,6 +236,7 @@ impl PyClassPyO3Options { ); set_option!(weakref); } + PyClassPyO3Option::Generic(generic) => set_option!(generic), } Ok(()) } @@ -418,6 +434,21 @@ fn impl_class( } } + let mut default_methods = descriptors_to_items( + cls, + args.options.rename_all.as_ref(), + args.options.frozen, + field_options, + ctx, + )?; + + let (default_class_geitem, default_class_geitem_method) = + pyclass_class_geitem(&args.options, &syn::parse_quote!(#cls), ctx)?; + + if let Some(default_class_geitem_method) = default_class_geitem_method { + default_methods.push(default_class_geitem_method); + } + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &syn::parse_quote!(#cls), ctx); @@ -432,21 +463,9 @@ fn impl_class( slots.extend(default_hash_slot); slots.extend(default_str_slot); - let py_class_impl = PyClassImplsBuilder::new( - cls, - args, - methods_type, - descriptors_to_items( - cls, - args.options.rename_all.as_ref(), - args.options.frozen, - field_options, - ctx, - )?, - slots, - ) - .doc(doc) - .impl_all(ctx)?; + let py_class_impl = PyClassImplsBuilder::new(cls, args, methods_type, default_methods, slots) + .doc(doc) + .impl_all(ctx)?; Ok(quote! { impl #pyo3_path::types::DerefToPyAny for #cls {} @@ -461,6 +480,7 @@ fn impl_class( #default_richcmp #default_hash #default_str + #default_class_geitem } }) } @@ -503,6 +523,10 @@ pub fn build_py_enum( bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); } + if let Some(generic) = &args.options.generic { + bail_spanned!(generic.span() => "enums do not support #[pyclass(generic)]"); + } + let doc = utils::get_doc(&enum_.attrs, None, ctx); let enum_ = PyClassEnum::new(enum_)?; impl_enum(enum_, &args, doc, method_type, ctx) @@ -1469,7 +1493,7 @@ fn generate_default_protocol_slot( slot.generate_type_slot( &syn::parse_quote!(#cls), &spec, - &format!("__default_{}__", name), + &format!("__default_{name}__"), ctx, ) } @@ -1997,6 +2021,46 @@ fn pyclass_hash( } } +fn pyclass_class_geitem( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + match options.generic { + Some(_) => { + let ident = format_ident!("__class_getitem__"); + let mut class_geitem_impl: syn::ImplItemFn = { + parse_quote! { + #[classmethod] + fn #ident<'py>( + cls: &#pyo3_path::Bound<'py, #pyo3_path::types::PyType>, + key: &#pyo3_path::Bound<'py, #pyo3_path::types::PyAny> + ) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::types::PyGenericAlias>> { + #pyo3_path::types::PyGenericAlias::new(cls.py(), cls.as_any(), key) + } + } + }; + + let spec = FnSpec::parse( + &mut class_geitem_impl.sig, + &mut class_geitem_impl.attrs, + Default::default(), + )?; + + let class_geitem_method = crate::pymethod::impl_py_method_def( + cls, + &spec, + &spec.get_doc(&class_geitem_impl.attrs, ctx), + Some(quote!(#pyo3_path::ffi::METH_CLASS)), + ctx, + )?; + Ok((Some(class_geitem_impl), Some(class_geitem_method))) + } + None => Ok((None, None)), + } +} + /// Implements most traits used by `#[pyclass]`. /// /// Specifically, it implements traits that only depend on class name, @@ -2145,6 +2209,7 @@ impl<'a> PyClassImplsBuilder<'a> { let is_subclass = self.attr.options.extends.is_some(); let is_mapping: bool = self.attr.options.mapping.is_some(); let is_sequence: bool = self.attr.options.sequence.is_some(); + let is_immutable_type = self.attr.options.immutable_type.is_some(); ensure_spanned!( !(is_mapping && is_sequence), @@ -2278,6 +2343,7 @@ impl<'a> PyClassImplsBuilder<'a> { const IS_SUBCLASS: bool = #is_subclass; const IS_MAPPING: bool = #is_mapping; const IS_SEQUENCE: bool = #is_sequence; + const IS_IMMUTABLE_TYPE: bool = #is_immutable_type; type BaseType = #base; type ThreadChecker = #thread_checker; @@ -2388,7 +2454,15 @@ impl<'a> PyClassImplsBuilder<'a> { fn impl_introspection(&self, ctx: &Ctx) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; let name = get_class_python_name(self.cls, self.attr).to_string(); - class_introspection_code(pyo3_path, self.cls, &name) + let ident = self.cls; + let static_introspection = class_introspection_code(pyo3_path, ident, &name); + let introspection_id = introspection_id_const(); + quote! { + #static_introspection + impl #ident { + #introspection_id + } + } } #[cfg(not(feature = "experimental-inspect"))] diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 68976190fbe..e512ca1cabc 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,5 +1,5 @@ #[cfg(feature = "experimental-inspect")] -use crate::introspection::function_introspection_code; +use crate::introspection::{function_introspection_code, introspection_id_const}; use crate::utils::Ctx; use crate::{ attributes::{ @@ -226,6 +226,26 @@ pub fn impl_wrap_pyfunction( FunctionSignature::from_arguments(arguments) }; + let vis = &func.vis; + let name = &func.sig.ident; + + #[cfg(feature = "experimental-inspect")] + let introspection = function_introspection_code( + pyo3_path, + Some(name), + &name.to_string(), + &signature, + None, + [] as [String; 0], + None, + ); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + let spec = method::FnSpec { tp, name: &func.sig.ident, @@ -237,16 +257,9 @@ pub fn impl_wrap_pyfunction( unsafety: func.sig.unsafety, }; - let vis = &func.vis; - let name = &func.sig.ident; - let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs, ctx), ctx); - #[cfg(feature = "experimental-inspect")] - let introspection = function_introspection_code(pyo3_path, &name.to_string()); - #[cfg(not(feature = "experimental-inspect"))] - let introspection = quote! {}; let wrapped_pyfunction = quote! { // Create a module with the same name as the `#[pyfunction]` - this way `use ` @@ -255,7 +268,7 @@ pub fn impl_wrap_pyfunction( #vis mod #name { pub(crate) struct MakeDef; pub const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::_PYO3_DEF; - #introspection + #introspection_id } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -269,6 +282,8 @@ pub fn impl_wrap_pyfunction( #[allow(non_snake_case)] #wrapper + + #introspection }; Ok(wrapped_pyfunction) } diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index deea3dfa052..fac1541bdf1 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -491,49 +491,11 @@ impl<'a> FunctionSignature<'a> { } fn default_value_for_parameter(&self, parameter: &str) -> String { - let mut default = "...".to_string(); if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name() == parameter) { - if let FnArg::Regular(RegularArg { - default_value: Some(arg_default), - .. - }) = fn_arg - { - match arg_default { - // literal values - syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { - syn::Lit::Str(s) => default = s.token().to_string(), - syn::Lit::Char(c) => default = c.token().to_string(), - syn::Lit::Int(i) => default = i.base10_digits().to_string(), - syn::Lit::Float(f) => default = f.base10_digits().to_string(), - syn::Lit::Bool(b) => { - default = if b.value() { - "True".to_string() - } else { - "False".to_string() - } - } - _ => {} - }, - // None - syn::Expr::Path(syn::ExprPath { - qself: None, path, .. - }) if path.is_ident("None") => { - default = "None".to_string(); - } - // others, unsupported yet so defaults to `...` - _ => {} - } - } else if let FnArg::Regular(RegularArg { - option_wrapped_type: Some(..), - .. - }) = fn_arg - { - // functions without a `#[pyo3(signature = (...))]` option - // will treat trailing `Option` arguments as having a default of `None` - default = "None".to_string(); - } + fn_arg.default_value() + } else { + "...".to_string() } - default } pub fn text_signature(&self, self_argument: Option<&str>) -> String { diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index 72f06721ec4..90b0d961cd8 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -1,14 +1,19 @@ use std::collections::HashSet; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::function_introspection_code; +#[cfg(feature = "experimental-inspect")] +use crate::method::{FnSpec, FnType}; use crate::utils::{has_attribute, has_attribute_with_namespace, Ctx, PyO3CratePath}; use crate::{ attributes::{take_pyo3_options, CrateAttribute}, konst::{ConstAttributes, ConstSpec}, pyfunction::PyFunctionOptions, - pymethod::{self, is_proto_method, MethodAndMethodDef, MethodAndSlotDef}, + pymethod::{ + self, is_proto_method, GeneratedPyMethod, MethodAndMethodDef, MethodAndSlotDef, PyMethod, + }, }; use proc_macro2::TokenStream; -use pymethod::GeneratedPyMethod; use quote::{format_ident, quote}; use syn::ImplItemFn; use syn::{ @@ -110,7 +115,7 @@ pub fn impl_methods( methods_type: PyClassMethodsType, options: PyImplOptions, ) -> syn::Result { - let mut trait_impls = Vec::new(); + let mut extra_fragments = Vec::new(); let mut proto_impls = Vec::new(); let mut methods = Vec::new(); let mut associated_methods = Vec::new(); @@ -125,9 +130,10 @@ pub fn impl_methods( fun_options.krate = fun_options.krate.or_else(|| options.krate.clone()); check_pyfunction(&ctx.pyo3_path, meth)?; - - match pymethod::gen_py_method(ty, &mut meth.sig, &mut meth.attrs, fun_options, ctx)? - { + let method = PyMethod::parse(&mut meth.sig, &mut meth.attrs, fun_options)?; + #[cfg(feature = "experimental-inspect")] + extra_fragments.push(method_introspection_code(&method.spec, ty, ctx)); + match pymethod::gen_py_method(ty, method, &meth.attrs, ctx)? { GeneratedPyMethod::Method(MethodAndMethodDef { associated_method, method_def, @@ -139,7 +145,7 @@ pub fn impl_methods( GeneratedPyMethod::SlotTraitImpl(method_name, token_stream) => { implemented_proto_fragments.insert(method_name); let attrs = get_cfg_attributes(&meth.attrs); - trait_impls.push(quote!(#(#attrs)* #token_stream)); + extra_fragments.push(quote!(#(#attrs)* #token_stream)); } GeneratedPyMethod::Proto(MethodAndSlotDef { associated_method, @@ -193,7 +199,7 @@ pub fn impl_methods( }; Ok(quote! { - #(#trait_impls)* + #(#extra_fragments)* #items @@ -336,3 +342,52 @@ pub(crate) fn get_cfg_attributes(attrs: &[syn::Attribute]) -> Vec<&syn::Attribut .filter(|attr| attr.path().is_ident("cfg")) .collect() } + +#[cfg(feature = "experimental-inspect")] +fn method_introspection_code(spec: &FnSpec<'_>, parent: &syn::Type, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + + // We introduce self/cls argument and setup decorators + let name = spec.python_name.to_string(); + let mut first_argument = None; + let mut decorators = Vec::new(); + match &spec.tp { + FnType::Getter(_) => { + first_argument = Some("self"); + decorators.push("property".into()); + } + FnType::Setter(_) => { + first_argument = Some("self"); + decorators.push(format!("{name}.setter")); + } + FnType::Fn(_) => { + first_argument = Some("self"); + } + FnType::FnNew | FnType::FnNewClass(_) => { + first_argument = Some("cls"); + } + FnType::FnClass(_) => { + first_argument = Some("cls"); + decorators.push("classmethod".into()); + } + FnType::FnStatic => { + decorators.push("staticmethod".into()); + } + FnType::FnModule(_) => (), // TODO: not sure this can happen + FnType::ClassAttribute => { + first_argument = Some("cls"); + // TODO: this combination only works with Python 3.9-3.11 https://docs.python.org/3.11/library/functions.html#classmethod + decorators.push("classmethod".into()); + decorators.push("property".into()); + } + } + function_introspection_code( + pyo3_path, + None, + &name, + &spec.signature, + first_argument, + decorators, + Some(parent), + ) +} diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index b4637b48012..434a96e9dde 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -4,8 +4,8 @@ use std::ffi::CString; use crate::attributes::{FromPyWithAttribute, NameAttribute, RenamingRule}; use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; use crate::params::{impl_regular_arg_param, Holders}; -use crate::utils::{deprecated_from_py_with, PythonDoc, TypeExt as _}; use crate::utils::{Ctx, LitCStr}; +use crate::utils::{PythonDoc, TypeExt as _}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, pyfunction::PyFunctionOptions, @@ -40,7 +40,7 @@ pub enum GeneratedPyMethod { pub struct PyMethod<'a> { kind: PyMethodKind, method_name: String, - spec: FnSpec<'a>, + pub spec: FnSpec<'a>, } enum PyMethodKind { @@ -160,11 +160,13 @@ enum PyMethodProtoKind { } impl<'a> PyMethod<'a> { - fn parse( + pub fn parse( sig: &'a mut syn::Signature, meth_attrs: &mut Vec, options: PyFunctionOptions, ) -> Result { + check_generic(sig)?; + ensure_function_options_valid(&options)?; let spec = FnSpec::parse(sig, meth_attrs, options)?; let method_name = spec.python_name.to_string(); @@ -187,14 +189,10 @@ pub fn is_proto_method(name: &str) -> bool { pub fn gen_py_method( cls: &syn::Type, - sig: &mut syn::Signature, - meth_attrs: &mut Vec, - options: PyFunctionOptions, + method: PyMethod<'_>, + meth_attrs: &[syn::Attribute], ctx: &Ctx, ) -> Result { - check_generic(sig)?; - ensure_function_options_valid(&options)?; - let method = PyMethod::parse(sig, meth_attrs, options)?; let spec = &method.spec; let Ctx { pyo3_path, .. } = ctx; @@ -278,7 +276,7 @@ pub fn gen_py_method( } pub fn check_generic(sig: &syn::Signature) -> syn::Result<()> { - let err_msg = |typ| format!("Python functions cannot have generic {} parameters", typ); + let err_msg = |typ| format!("Python functions cannot have generic {typ} parameters"); for param in &sig.generics.params { match param { syn::GenericParam::Lifetime(_) => {} @@ -659,10 +657,8 @@ pub fn impl_py_setter_def( let (from_py_with, ident) = if let Some(from_py_with) = &value_arg.from_py_with().as_ref().map(|f| &f.value) { let ident = syn::Ident::new("from_py_with", from_py_with.span()); - let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); ( quote_spanned! { from_py_with.span() => - #d let #ident = #from_py_with; }, ident, @@ -1603,7 +1599,7 @@ fn extract_proto_arguments( if let FnArg::Py(..) = arg { args.push(quote! { py }); } else { - let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); + let ident = syn::Ident::new(&format!("arg{non_python_args}"), Span::call_site()); let conversions = proto_args.get(non_python_args) .ok_or_else(|| err_spanned!(arg.ty().span() => format!("Expected at most {} non-python arguments", proto_args.len())))? .extract(&ident, arg, extract_error_mode, holders, ctx); diff --git a/pyo3-macros-backend/src/pyversions.rs b/pyo3-macros-backend/src/pyversions.rs index 4c0998667d8..3c5ac47fb84 100644 --- a/pyo3-macros-backend/src/pyversions.rs +++ b/pyo3-macros-backend/src/pyversions.rs @@ -2,5 +2,10 @@ use pyo3_build_config::PythonVersion; pub fn is_abi3_before(major: u8, minor: u8) -> bool { let config = pyo3_build_config::get(); - config.abi3 && config.version < PythonVersion { major, minor } + config.abi3 && !config.is_free_threaded() && config.version < PythonVersion { major, minor } +} + +pub fn is_py_before(major: u8, minor: u8) -> bool { + let config = pyo3_build_config::get(); + config.version < PythonVersion { major, minor } } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index bdec23388df..09f86158834 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,6 +1,6 @@ -use crate::attributes::{CrateAttribute, ExprPathWrap, RenamingRule}; +use crate::attributes::{CrateAttribute, RenamingRule}; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned, ToTokens}; +use quote::{quote, ToTokens}; use std::ffi::CString; use syn::spanned::Spanned; use syn::{punctuated::Punctuated, Token}; @@ -324,20 +324,6 @@ pub(crate) fn has_attribute_with_namespace( }) } -pub(crate) fn deprecated_from_py_with(expr_path: &ExprPathWrap) -> Option { - let path = quote!(#expr_path).to_string(); - let msg = - format!("remove the quotes from the literal\n= help: use `{path}` instead of `\"{path}\"`"); - expr_path.from_lit_str.then(|| { - quote_spanned! { expr_path.span() => - #[deprecated(since = "0.24.0", note = #msg)] - #[allow(dead_code)] - const LIT_STR_DEPRECATION: () = (); - let _: () = LIT_STR_DEPRECATION; - } - }) -} - pub(crate) trait TypeExt { /// Replaces all explicit lifetimes in `self` with elided (`'_`) lifetimes /// diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index c7bd43fad69..7df15c23f76 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.25.0-dev" +version = "0.25.0" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -23,7 +23,7 @@ experimental-inspect = ["pyo3-macros-backend/experimental-inspect"] proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.25.0-dev" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.25.0" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 1ff0f0332de..adfe3a27348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.25.0-dev" +version = "0.25.0" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/pytests/pytest.ini b/pytests/pytest.ini new file mode 100644 index 00000000000..3d62037f722 --- /dev/null +++ b/pytests/pytest.ini @@ -0,0 +1,3 @@ +# see https://github.com/PyO3/pyo3/issues/5094 for details +[pytest] +filterwarnings = ignore::DeprecationWarning:pytest_asyncio.* \ No newline at end of file diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 1091e6c16b3..4e681b2c941 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -80,8 +80,7 @@ impl AssertingBaseClass { fn new(cls: &Bound<'_, PyType>, expected_type: Bound<'_, PyType>) -> PyResult { if !cls.is(&expected_type) { return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type + "{cls:?} != {expected_type:?}" ))); } Ok(Self) @@ -104,6 +103,45 @@ impl ClassWithDict { } } +#[pyclass] +struct ClassWithDecorators { + attr: usize, +} + +#[pymethods] +impl ClassWithDecorators { + #[new] + #[classmethod] + fn new(_cls: Bound<'_, PyType>) -> Self { + Self { attr: 0 } + } + + #[getter] + fn get_attr(&self) -> usize { + self.attr + } + + #[setter] + fn set_attr(&mut self, value: usize) { + self.attr = value; + } + + #[classmethod] + fn cls_method(_cls: &Bound<'_, PyType>) -> usize { + 1 + } + + #[staticmethod] + fn static_method() -> usize { + 2 + } + + #[classattr] + fn cls_attribute() -> usize { + 3 + } +} + #[pymodule(gil_used = false)] pub mod pyclasses { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] @@ -111,6 +149,7 @@ pub mod pyclasses { use super::ClassWithDict; #[pymodule_export] use super::{ - AssertingBaseClass, ClassWithoutConstructor, EmptyClass, PyClassIter, PyClassThreadIter, + AssertingBaseClass, ClassWithDecorators, ClassWithoutConstructor, EmptyClass, PyClassIter, + PyClassThreadIter, }; } diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index 024641d3d2e..19e30712909 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -67,13 +67,21 @@ fn args_kwargs<'py>( (args, kwargs) } -#[pymodule(gil_used = false)] -pub fn pyfunctions(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(none, m)?)?; - m.add_function(wrap_pyfunction!(simple, m)?)?; - m.add_function(wrap_pyfunction!(simple_args, m)?)?; - m.add_function(wrap_pyfunction!(simple_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(simple_args_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(args_kwargs, m)?)?; - Ok(()) +#[pyfunction(signature = (a, /, b))] +fn positional_only<'py>(a: Any<'py>, b: Any<'py>) -> (Any<'py>, Any<'py>) { + (a, b) +} + +#[pyfunction(signature = (a = false, b = 0, c = 0.0, d = ""))] +fn with_typed_args(a: bool, b: u64, c: f64, d: &str) -> (bool, u64, f64, &str) { + (a, b, c, d) +} + +#[pymodule] +pub mod pyfunctions { + #[pymodule_export] + use super::{ + args_kwargs, none, positional_only, simple, simple_args, simple_args_kwargs, simple_kwargs, + with_typed_args, + }; } diff --git a/pytests/stubs/pyclasses.pyi b/pytests/stubs/pyclasses.pyi index 2a36e0b4540..13b52f1c4e2 100644 --- a/pytests/stubs/pyclasses.pyi +++ b/pytests/stubs/pyclasses.pyi @@ -1,5 +1,31 @@ -class AssertingBaseClass: ... +class AssertingBaseClass: + def __new__(cls, /, expected_type): ... + +class ClassWithDecorators: + def __new__(cls, /): ... + @property + def attr(self, /): ... + @attr.setter + def attr(self, /, value): ... + @classmethod + @property + def cls_attribute(cls, /): ... + @classmethod + def cls_method(cls, /): ... + @staticmethod + def static_method(): ... + class ClassWithoutConstructor: ... -class EmptyClass: ... -class PyClassIter: ... -class PyClassThreadIter: ... + +class EmptyClass: + def __len__(self, /): ... + def __new__(cls, /): ... + def method(self, /): ... + +class PyClassIter: + def __new__(cls, /): ... + def __next__(self, /): ... + +class PyClassThreadIter: + def __new__(cls, /): ... + def __next__(self, /): ... diff --git a/pytests/stubs/pyfunctions.pyi b/pytests/stubs/pyfunctions.pyi index e69de29bb2d..5fb5e6c474c 100644 --- a/pytests/stubs/pyfunctions.pyi +++ b/pytests/stubs/pyfunctions.pyi @@ -0,0 +1,8 @@ +def args_kwargs(*args, **kwargs): ... +def none(): ... +def positional_only(a, /, b): ... +def simple(a, b=None, *, c=None): ... +def simple_args(a, b=None, *args, c=None): ... +def simple_args_kwargs(a, b=None, *args, c=None, **kwargs): ... +def simple_kwargs(a, b=None, c=None, **kwargs): ... +def with_typed_args(a=False, b=0, c=0.0, d=""): ... diff --git a/pytests/tests/test_pyclasses.py b/pytests/tests/test_pyclasses.py index 9f611b634b6..a641c9770a4 100644 --- a/pytests/tests/test_pyclasses.py +++ b/pytests/tests/test_pyclasses.py @@ -121,3 +121,32 @@ def test_dict(): d.foo = 42 assert d.__dict__ == {"foo": 42} + + +def test_getter(benchmark): + obj = pyclasses.ClassWithDecorators() + benchmark(lambda: obj.attr) + + +def test_setter(benchmark): + obj = pyclasses.ClassWithDecorators() + + def set_attr(): + obj.attr = 42 + + benchmark(set_attr) + + +def test_class_attribute(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.cls_attribute) + + +def test_class_method(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.cls_method()) + + +def test_static_method(benchmark): + cls = pyclasses.ClassWithDecorators + benchmark(lambda: cls.static_method()) diff --git a/pytests/tests/test_pyfunctions.py b/pytests/tests/test_pyfunctions.py index c6fb448248b..b3897f289c6 100644 --- a/pytests/tests/test_pyfunctions.py +++ b/pytests/tests/test_pyfunctions.py @@ -1,3 +1,5 @@ +from typing import Tuple + from pyo3_pytests import pyfunctions @@ -58,7 +60,7 @@ def test_simple_kwargs_rs(benchmark): def simple_args_kwargs_py(a, b=None, *args, c=None, **kwargs): - return (a, b, args, c, kwargs) + return a, b, args, c, kwargs def test_simple_args_kwargs_py(benchmark): @@ -72,7 +74,7 @@ def test_simple_args_kwargs_rs(benchmark): def args_kwargs_py(*args, **kwargs): - return (args, kwargs) + return args, kwargs def test_args_kwargs_py(benchmark): @@ -83,3 +85,36 @@ def test_args_kwargs_rs(benchmark): rust = benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, bar=4, foo=10) py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10) assert rust == py + + +# TODO: the second argument should be positional-only +# but can't be without breaking tests on Python 3.7. +# See gh-5095. +def positional_only_py(a, b): + return a, b + + +def test_positional_only_py(benchmark): + benchmark(positional_only_py, 1, "foo") + + +def test_positional_only_rs(benchmark): + rust = benchmark(pyfunctions.positional_only, 1, "foo") + py = positional_only_py(1, "foo") + assert rust == py + + +def with_typed_args_py( + a: bool, b: int, c: float, d: str +) -> Tuple[bool, int, float, str]: + return a, b, c, d + + +def test_with_typed_args_py(benchmark): + benchmark(with_typed_args_py, True, 1, 1.2, "foo") + + +def test_with_typed_args_rs(benchmark): + rust = benchmark(pyfunctions.with_typed_args, True, 1, 1.2, "foo") + py = with_typed_args_py(True, 1, 1.2, "foo") + assert rust == py diff --git a/src/buffer.rs b/src/buffer.rs index c5e5de568eb..6f74b698de7 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -702,7 +702,7 @@ mod tests { buffer.0.suboffsets, buffer.0.internal ); - let debug_repr = format!("{:?}", buffer); + let debug_repr = format!("{buffer:?}"); assert_eq!(debug_repr, expected); }); } @@ -829,8 +829,7 @@ mod tests { assert_eq!( ElementType::from_format(cstr), expected, - "element from format &Cstr: {:?}", - cstr, + "element from format &Cstr: {cstr:?}", ); } } diff --git a/src/conversion.rs b/src/conversion.rs index 53c9f3a9de8..165175fae54 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -5,67 +5,9 @@ use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{ - ffi, Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyRefMut, Python, -}; +use crate::{Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyRefMut, Python}; use std::convert::Infallible; -/// Returns a borrowed pointer to a Python object. -/// -/// The returned pointer will be valid for as long as `self` is. It may be null depending on the -/// implementation. -/// -/// # Examples -/// -/// ```rust -/// use pyo3::prelude::*; -/// use pyo3::ffi; -/// -/// Python::with_gil(|py| { -/// let s = "foo".into_pyobject(py)?; -/// let ptr = s.as_ptr(); -/// -/// let is_really_a_pystring = unsafe { ffi::PyUnicode_CheckExact(ptr) }; -/// assert_eq!(is_really_a_pystring, 1); -/// # Ok::<_, PyErr>(()) -/// }) -/// # .unwrap(); -/// ``` -/// -/// # Safety -/// -/// For callers, it is your responsibility to make sure that the underlying Python object is not dropped too -/// early. For example, the following code will cause undefined behavior: -/// -/// ```rust,no_run -/// # use pyo3::prelude::*; -/// # use pyo3::ffi; -/// # -/// Python::with_gil(|py| { -/// // ERROR: calling `.as_ptr()` will throw away the temporary object and leave `ptr` dangling. -/// let ptr: *mut ffi::PyObject = 0xabad1dea_u32.into_pyobject(py)?.as_ptr(); -/// -/// let isnt_a_pystring = unsafe { -/// // `ptr` is dangling, this is UB -/// ffi::PyUnicode_CheckExact(ptr) -/// }; -/// # assert_eq!(isnt_a_pystring, 0); -/// # Ok::<_, PyErr>(()) -/// }) -/// # .unwrap(); -/// ``` -/// -/// This happens because the pointer returned by `as_ptr` does not carry any lifetime information -/// and the Python object is dropped immediately after the `0xabad1dea_u32.into_pyobject(py).as_ptr()` -/// expression is evaluated. To fix the problem, bind Python object to a local variable like earlier -/// to keep the Python object alive until the end of its scope. -/// -/// Implementors must ensure this returns a valid pointer to a Python object, which borrows a reference count from `&self`. -pub unsafe trait AsPyPointer { - /// Returns the underlying FFI pointer as a borrowed pointer. - fn as_ptr(&self) -> *mut ffi::PyObject; -} - /// Defines a conversion from a Rust type to a Python object, which may fail. /// /// This trait has `#[derive(IntoPyObject)]` to automatically implement it for simple types and diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index d2cb3f3eb60..0bf346d835a 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -113,7 +113,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -141,7 +141,7 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { @@ -160,7 +160,7 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { diff --git a/src/conversions/bigdecimal.rs b/src/conversions/bigdecimal.rs index 446e38bf6c5..129def772c8 100644 --- a/src/conversions/bigdecimal.rs +++ b/src/conversions/bigdecimal.rs @@ -161,8 +161,7 @@ mod test_bigdecimal { locals.set_item("rs_dec", &rs_dec).unwrap(); py.run( &CString::new(format!( - "import decimal\npy_dec = decimal.Decimal(\"{}\")\nassert py_dec == rs_dec", - num)).unwrap(), + "import decimal\npy_dec = decimal.Decimal(\"{num}\")\nassert py_dec == rs_dec")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: BigDecimal = rs_dec.extract().unwrap(); assert_eq!(num, roundtripped); diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index fb8890d696a..bf99951e459 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -47,13 +47,10 @@ use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; use crate::intern; use crate::types::any::PyAnyMethods; use crate::types::PyNone; -use crate::types::{ - datetime::timezone_from_offset, timezone_utc, PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, - PyTzInfoAccess, -}; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; #[cfg(not(Py_LIMITED_API))] use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; -use crate::{ffi, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyResult, Python}; +use crate::{ffi, Borrowed, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyResult, Python}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, Offset, @@ -348,8 +345,7 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime Err(PyValueError::new_err(format!( - "The datetime {:?} contains an incompatible timezone", - dt + "The datetime {dt:?} contains an incompatible timezone" ))), } } @@ -363,7 +359,7 @@ impl<'py> IntoPyObject<'py> for FixedOffset { fn into_pyobject(self, py: Python<'py>) -> Result { let seconds_offset = self.local_minus_utc(); let td = PyDelta::new(py, 0, seconds_offset, 0, true)?; - timezone_from_offset(&td) + PyTzInfo::fixed_offset(py, td) } } @@ -394,8 +390,7 @@ impl FromPyObject<'_> for FixedOffset { let py_timedelta = ob.call_method1("utcoffset", (PyNone::get(ob.py()),))?; if py_timedelta.is_none() { return Err(PyTypeError::new_err(format!( - "{:?} is not a fixed offset timezone", - ob + "{ob:?} is not a fixed offset timezone" ))); } let total_seconds: Duration = py_timedelta.extract()?; @@ -408,17 +403,17 @@ impl FromPyObject<'_> for FixedOffset { impl<'py> IntoPyObject<'py> for Utc { type Target = PyTzInfo; - type Output = Bound<'py, Self::Target>; + type Output = Borrowed<'static, 'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - Ok(timezone_utc(py)) + PyTzInfo::utc(py) } } impl<'py> IntoPyObject<'py> for &Utc { type Target = PyTzInfo; - type Output = Bound<'py, Self::Target>; + type Output = Borrowed<'static, 'py, Self::Target>; type Error = PyErr; #[inline] @@ -429,7 +424,7 @@ impl<'py> IntoPyObject<'py> for &Utc { impl FromPyObject<'_> for Utc { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - let py_utc = timezone_utc(ob.py()); + let py_utc = PyTzInfo::utc(ob.py())?; if ob.eq(py_utc)? { Ok(Utc) } else { @@ -662,10 +657,7 @@ mod tests { let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); assert!( delta.eq(&py_delta).unwrap(), - "{}: {} != {}", - name, - delta, - py_delta + "{name}: {delta} != {py_delta}" ); }); }; @@ -701,7 +693,7 @@ mod tests { Python::with_gil(|py| { let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); let py_delta: Duration = py_delta.extract().unwrap(); - assert_eq!(py_delta, delta, "{}: {} != {}", name, py_delta, delta); + assert_eq!(py_delta, delta, "{name}: {py_delta} != {delta}"); }) }; @@ -764,10 +756,7 @@ mod tests { assert_eq!( date.compare(&py_date).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - date, - py_date + "{name}: {date} != {py_date}" ); }) }; @@ -785,7 +774,7 @@ mod tests { let py_date = new_py_datetime_ob(py, "date", (year, month, day)); let py_date: NaiveDate = py_date.extract().unwrap(); let date = NaiveDate::from_ymd_opt(year, month, day).unwrap(); - assert_eq!(py_date, date, "{}: {} != {}", name, date, py_date); + assert_eq!(py_date, date, "{name}: {date} != {py_date}"); }) }; @@ -823,10 +812,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -866,10 +852,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -926,7 +909,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc(py); + let tz_utc = PyTzInfo::utc(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -1056,13 +1039,7 @@ mod tests { .into_pyobject(py) .unwrap(); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); - assert!( - time.eq(&py_time).unwrap(), - "{}: {} != {}", - name, - time, - py_time - ); + assert!(time.eq(&py_time).unwrap(), "{name}: {time} != {py_time}"); }; check_time("regular", 3, 5, 7, 999_999, 999_999); @@ -1145,7 +1122,7 @@ mod tests { Python::with_gil(|py| { let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).unwrap(); - let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); + let code = format!("datetime.datetime.fromtimestamp({timestamp}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={timedelta})))"); let t = py.eval(&CString::new(code).unwrap(), Some(&globals), None).unwrap(); // Get ISO 8601 string from python diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index d564bcca87b..e8e5dc60e7a 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -37,27 +37,23 @@ use crate::conversion::IntoPyObject; use crate::exceptions::PyValueError; use crate::pybacked::PyBackedStr; -use crate::sync::GILOnceCell; -use crate::types::{any::PyAnyMethods, PyType}; -use crate::{intern, Bound, FromPyObject, Py, PyAny, PyErr, PyResult, Python}; +use crate::types::{any::PyAnyMethods, PyTzInfo}; +use crate::{intern, Bound, FromPyObject, PyAny, PyErr, PyResult, Python}; use chrono_tz::Tz; use std::str::FromStr; impl<'py> IntoPyObject<'py> for Tz { - type Target = PyAny; + type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); - ZONE_INFO - .import(py, "zoneinfo", "ZoneInfo") - .and_then(|obj| obj.call1((self.name(),))) + PyTzInfo::timezone(py, self.name()) } } impl<'py> IntoPyObject<'py> for &Tz { - type Target = PyAny; + type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -81,6 +77,7 @@ impl FromPyObject<'_> for Tz { mod tests { use super::*; use crate::prelude::PyAnyMethods; + use crate::types::PyTzInfo; use crate::Python; use chrono::{DateTime, Utc}; use chrono_tz::Tz; @@ -152,8 +149,8 @@ mod tests { #[cfg(not(Py_GIL_DISABLED))] // https://github.com/python/cpython/issues/116738#issuecomment-2404360445 fn test_into_pyobject() { Python::with_gil(|py| { - let assert_eq = |l: Bound<'_, PyAny>, r: Bound<'_, PyAny>| { - assert!(l.eq(&r).unwrap(), "{:?} != {:?}", l, r); + let assert_eq = |l: Bound<'_, PyTzInfo>, r: Bound<'_, PyTzInfo>| { + assert!(l.eq(&r).unwrap(), "{l:?} != {r:?}"); }; assert_eq( @@ -168,11 +165,7 @@ mod tests { }); } - fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyAny> { - zoneinfo_class(py).call1((name,)).unwrap() - } - - fn zoneinfo_class(py: Python<'_>) -> Bound<'_, PyAny> { - py.import("zoneinfo").unwrap().getattr("ZoneInfo").unwrap() + fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyTzInfo> { + PyTzInfo::timezone(py, name).unwrap() } } diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index 42d7a12c872..4a501de9c69 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -119,7 +119,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -147,7 +147,7 @@ mod tests { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { @@ -166,7 +166,7 @@ mod tests { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { diff --git a/src/conversions/jiff.rs b/src/conversions/jiff.rs index 2b90686844a..814040e5fb4 100644 --- a/src/conversions/jiff.rs +++ b/src/conversions/jiff.rs @@ -48,15 +48,11 @@ //! ``` use crate::exceptions::{PyTypeError, PyValueError}; use crate::pybacked::PyBackedStr; -use crate::sync::GILOnceCell; -use crate::types::{ - datetime::timezone_from_offset, timezone_utc, PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, - PyTzInfoAccess, -}; -use crate::types::{PyAnyMethods, PyNone, PyType}; +use crate::types::{PyAnyMethods, PyNone}; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; #[cfg(not(Py_LIMITED_API))] use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; -use crate::{intern, Bound, FromPyObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python}; +use crate::{intern, Bound, FromPyObject, IntoPyObject, PyAny, PyErr, PyResult, Python}; use jiff::civil::{Date, DateTime, Time}; use jiff::tz::{Offset, TimeZone}; use jiff::{SignedDuration, Span, Timestamp, Zoned}; @@ -278,6 +274,7 @@ impl<'py> IntoPyObject<'py> for &Zoned { }; Some(zoned.timestamp() + (zoned.offset() - prev.offset()) <= start_of_current_offset) } + datetime_to_pydatetime( py, &self.datetime(), @@ -333,17 +330,14 @@ impl<'py> IntoPyObject<'py> for &TimeZone { fn into_pyobject(self, py: Python<'py>) -> Result { if self == &TimeZone::UTC { - Ok(timezone_utc(py)) - } else if let Some(iana_name) = self.iana_name() { - static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); - let tz = ZONE_INFO - .import(py, "zoneinfo", "ZoneInfo") - .and_then(|obj| obj.call1((iana_name,)))? - .downcast_into()?; - Ok(tz) - } else { - self.to_fixed_offset()?.into_pyobject(py) + return Ok(PyTzInfo::utc(py)?.to_owned()); + } + + if let Some(iana_name) = self.iana_name() { + return PyTzInfo::timezone(py, iana_name); } + + self.to_fixed_offset()?.into_pyobject(py) } } @@ -367,12 +361,10 @@ impl<'py> IntoPyObject<'py> for &Offset { fn into_pyobject(self, py: Python<'py>) -> Result { if self == &Offset::UTC { - return Ok(timezone_utc(py)); + return Ok(PyTzInfo::utc(py)?.to_owned()); } - let delta = self.duration_since(Offset::UTC).into_pyobject(py)?; - - timezone_from_offset(&delta) + PyTzInfo::fixed_offset(py, self.duration_since(Offset::UTC)) } } @@ -394,8 +386,7 @@ impl<'py> FromPyObject<'py> for Offset { let py_timedelta = ob.call_method1(intern!(py, "utcoffset"), (PyNone::get(py),))?; if py_timedelta.is_none() { return Err(PyTypeError::new_err(format!( - "{:?} is not a fixed offset timezone", - ob + "{ob:?} is not a fixed offset timezone" ))); } @@ -476,8 +467,6 @@ impl From for PyErr { #[cfg(test)] mod tests { use super::*; - #[cfg(not(Py_LIMITED_API))] - use crate::types::timezone_utc; use crate::{types::PyTuple, BoundObject}; use jiff::tz::Offset; use std::cmp::Ordering; @@ -586,10 +575,7 @@ mod tests { assert_eq!( date.compare(&py_date).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - date, - py_date + "{name}: {date} != {py_date}" ); }) }; @@ -607,7 +593,7 @@ mod tests { let py_date = new_py_datetime_ob(py, "date", (year, month, day)); let py_date: Date = py_date.extract().unwrap(); let date = Date::new(year, month, day).unwrap(); - assert_eq!(py_date, date, "{}: {} != {}", name, date, py_date); + assert_eq!(py_date, date, "{name}: {date} != {py_date}"); }) }; @@ -644,10 +630,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -663,7 +646,7 @@ mod tests { let offset = Offset::from_seconds(3600).unwrap(); let datetime = DateTime::new(year, month, day, hour, minute, second, ms * 1000) .map_err(|e| { - eprintln!("{}: {}", name, e); + eprintln!("{name}: {e}"); e }) .unwrap() @@ -679,10 +662,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -727,7 +707,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc(py); + let tz_utc = PyTzInfo::utc(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -893,13 +873,7 @@ mod tests { .into_pyobject(py) .unwrap(); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); - assert!( - time.eq(&py_time).unwrap(), - "{}: {} != {}", - name, - time, - py_time - ); + assert!(time.eq(&py_time).unwrap(), "{name}: {time} != {py_time}"); }; check_time("regular", 3, 5, 7, 999_999, 999_999); @@ -1001,7 +975,7 @@ mod tests { Python::with_gil(|py| { let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).unwrap(); - let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); + let code = format!("datetime.datetime.fromtimestamp({timestamp}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={timedelta})))"); let t = py.eval(&CString::new(code).unwrap(), Some(&globals), None).unwrap(); // Get ISO 8601 string from python diff --git a/src/conversions/mod.rs b/src/conversions/mod.rs index 0afd39745c9..c623ca7b379 100644 --- a/src/conversions/mod.rs +++ b/src/conversions/mod.rs @@ -12,8 +12,10 @@ pub mod jiff; pub mod num_bigint; pub mod num_complex; pub mod num_rational; +pub mod ordered_float; pub mod rust_decimal; pub mod serde; pub mod smallvec; mod std; +pub mod time; pub mod uuid; diff --git a/src/conversions/ordered_float.rs b/src/conversions/ordered_float.rs new file mode 100644 index 00000000000..79bce2c66c1 --- /dev/null +++ b/src/conversions/ordered_float.rs @@ -0,0 +1,330 @@ +#![cfg(feature = "ordered-float")] +//! Conversions to and from [ordered-float](https://docs.rs/ordered-float) types. +//! [`NotNan`]`<`[`f32`]`>` and [`NotNan`]`<`[`f64`]`>`. +//! [`OrderedFloat`]`<`[`f32`]`>` and [`OrderedFloat`]`<`[`f64`]`>`. +//! +//! This is useful for converting between Python's float into and from a native Rust type. +//! +//! Take care when comparing sorted collections of float types between Python and Rust. +//! They will likely differ due to the ambiguous sort order of NaNs in Python. +// +//! +//! To use this feature, add to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"ordered-float\"] }")] +//! ordered-float = "5.0.0" +//! ``` +//! +//! # Example +//! +//! Rust code to create functions that add ordered floats: +//! +//! ```rust,no_run +//! use ordered_float::{NotNan, OrderedFloat}; +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn add_not_nans(a: NotNan, b: NotNan) -> NotNan { +//! a + b +//! } +//! +//! #[pyfunction] +//! fn add_ordered_floats(a: OrderedFloat, b: OrderedFloat) -> OrderedFloat { +//! a + b +//! } +//! +//! #[pymodule] +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(add_not_nans, m)?)?; +//! m.add_function(wrap_pyfunction!(add_ordered_floats, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Python code that validates the functionality: +//! ```python +//! from my_module import add_not_nans, add_ordered_floats +//! +//! assert add_not_nans(1.0,2.0) == 3.0 +//! assert add_ordered_floats(1.0,2.0) == 3.0 +//! ``` + +use crate::conversion::IntoPyObject; +use crate::exceptions::PyValueError; +use crate::types::{any::PyAnyMethods, PyFloat}; +use crate::{Bound, FromPyObject, PyAny, PyResult, Python}; +use ordered_float::{NotNan, OrderedFloat}; +use std::convert::Infallible; + +macro_rules! float_conversions { + ($wrapper:ident, $float_type:ty, $constructor:expr) => { + impl FromPyObject<'_> for $wrapper<$float_type> { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + let val: $float_type = obj.extract()?; + $constructor(val) + } + } + + impl<'py> IntoPyObject<'py> for $wrapper<$float_type> { + type Target = PyFloat; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + self.into_inner().into_pyobject(py) + } + } + + impl<'py> IntoPyObject<'py> for &$wrapper<$float_type> { + type Target = PyFloat; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) + } + } + }; +} +float_conversions!(OrderedFloat, f32, |val| Ok(OrderedFloat(val))); +float_conversions!(OrderedFloat, f64, |val| Ok(OrderedFloat(val))); +float_conversions!(NotNan, f32, |val| NotNan::new(val) + .map_err(|e| PyValueError::new_err(e.to_string()))); +float_conversions!(NotNan, f64, |val| NotNan::new(val) + .map_err(|e| PyValueError::new_err(e.to_string()))); + +#[cfg(test)] +mod test_ordered_float { + use super::*; + use crate::ffi::c_str; + use crate::py_run; + + #[cfg(not(target_arch = "wasm32"))] + use proptest::prelude::*; + + macro_rules! float_roundtrip_tests { + ($wrapper:ident, $float_type:ty, $constructor:expr, $standard_test:ident, $wasm_test:ident, $infinity_test:ident, $zero_test:ident) => { + #[cfg(not(target_arch = "wasm32"))] + proptest! { + #[test] + fn $standard_test(inner_f: $float_type) { + let f = $constructor(inner_f); + + Python::with_gil(|py| { + let f_py: Bound<'_, PyFloat> = f.into_pyobject(py).unwrap(); + + py_run!( + py, + f_py, + &format!( + "import math\nassert math.isclose(f_py, {})", + inner_f as f64 // Always interpret the literal rs float value as f64 + // so that it's comparable with the python float + ) + ); + + let roundtripped_f: $wrapper<$float_type> = f_py.extract().unwrap(); + + assert_eq!(f, roundtripped_f); + }) + } + } + + #[cfg(target_arch = "wasm32")] + #[test] + fn $wasm_test() { + let inner_f = 10.0; + let f = $constructor(inner_f); + + Python::with_gil(|py| { + let f_py: Bound<'_, PyFloat> = f.into_pyobject(py).unwrap(); + + py_run!( + py, + f_py, + &format!( + "import math\nassert math.isclose(f_py, {})", + inner_f as f64 // Always interpret the literal rs float value as f64 + // so that it's comparable with the python float + ) + ); + + let roundtripped_f: $wrapper<$float_type> = f_py.extract().unwrap(); + + assert_eq!(f, roundtripped_f); + }) + } + + #[test] + fn $infinity_test() { + let inner_pinf = <$float_type>::INFINITY; + let pinf = $constructor(inner_pinf); + + let inner_ninf = <$float_type>::NEG_INFINITY; + let ninf = $constructor(inner_ninf); + + Python::with_gil(|py| { + let pinf_py: Bound<'_, PyFloat> = pinf.into_pyobject(py).unwrap(); + let ninf_py: Bound<'_, PyFloat> = ninf.into_pyobject(py).unwrap(); + + py_run!( + py, + pinf_py ninf_py, + "\ + assert pinf_py == float('inf')\n\ + assert ninf_py == float('-inf')" + ); + + let roundtripped_pinf: $wrapper<$float_type> = pinf_py.extract().unwrap(); + let roundtripped_ninf: $wrapper<$float_type> = ninf_py.extract().unwrap(); + + assert_eq!(pinf, roundtripped_pinf); + assert_eq!(ninf, roundtripped_ninf); + }) + } + + #[test] + fn $zero_test() { + let inner_pzero: $float_type = 0.0; + let pzero = $constructor(inner_pzero); + + let inner_nzero: $float_type = -0.0; + let nzero = $constructor(inner_nzero); + + Python::with_gil(|py| { + let pzero_py: Bound<'_, PyFloat> = pzero.into_pyobject(py).unwrap(); + let nzero_py: Bound<'_, PyFloat> = nzero.into_pyobject(py).unwrap(); + + // This python script verifies that the values are 0.0 in magnitude + // and that the signs are correct(+0.0 vs -0.0) + py_run!( + py, + pzero_py nzero_py, + "\ + import math\n\ + assert pzero_py == 0.0\n\ + assert math.copysign(1.0, pzero_py) > 0.0\n\ + assert nzero_py == 0.0\n\ + assert math.copysign(1.0, nzero_py) < 0.0" + ); + + let roundtripped_pzero: $wrapper<$float_type> = pzero_py.extract().unwrap(); + let roundtripped_nzero: $wrapper<$float_type> = nzero_py.extract().unwrap(); + + assert_eq!(pzero, roundtripped_pzero); + assert_eq!(roundtripped_pzero.signum(), 1.0); + assert_eq!(nzero, roundtripped_nzero); + assert_eq!(roundtripped_nzero.signum(), -1.0); + }) + } + }; + } + float_roundtrip_tests!( + OrderedFloat, + f32, + OrderedFloat, + ordered_float_f32_standard, + ordered_float_f32_wasm, + ordered_float_f32_infinity, + ordered_float_f32_zero + ); + float_roundtrip_tests!( + OrderedFloat, + f64, + OrderedFloat, + ordered_float_f64_standard, + ordered_float_f64_wasm, + ordered_float_f64_infinity, + ordered_float_f64_zero + ); + float_roundtrip_tests!( + NotNan, + f32, + |val| NotNan::new(val).unwrap(), + not_nan_f32_standard, + not_nan_f32_wasm, + not_nan_f32_infinity, + not_nan_f32_zero + ); + float_roundtrip_tests!( + NotNan, + f64, + |val| NotNan::new(val).unwrap(), + not_nan_f64_standard, + not_nan_f64_wasm, + not_nan_f64_infinity, + not_nan_f64_zero + ); + + macro_rules! ordered_float_pynan_tests { + ($test_name:ident, $float_type:ty) => { + #[test] + fn $test_name() { + let inner_nan: $float_type = <$float_type>::NAN; + let nan = OrderedFloat(inner_nan); + + Python::with_gil(|py| { + let nan_py: Bound<'_, PyFloat> = nan.into_pyobject(py).unwrap(); + + py_run!( + py, + nan_py, + "\ + import math\n\ + assert math.isnan(nan_py)" + ); + + let roundtripped_nan: OrderedFloat<$float_type> = nan_py.extract().unwrap(); + + assert_eq!(nan, roundtripped_nan); + }) + } + }; + } + ordered_float_pynan_tests!(test_ordered_float_pynan_f32, f32); + ordered_float_pynan_tests!(test_ordered_float_pynan_f64, f64); + + macro_rules! not_nan_pynan_tests { + ($test_name:ident, $float_type:ty) => { + #[test] + fn $test_name() { + Python::with_gil(|py| { + let nan_py = py.eval(c_str!("float('nan')"), None, None).unwrap(); + + let nan_rs: PyResult> = nan_py.extract(); + + assert!(nan_rs.is_err()); + }) + } + }; + } + not_nan_pynan_tests!(test_not_nan_pynan_f32, f32); + not_nan_pynan_tests!(test_not_nan_pynan_f64, f64); + + macro_rules! py64_rs32 { + ($test_name:ident, $wrapper:ident, $float_type:ty) => { + #[test] + fn $test_name() { + Python::with_gil(|py| { + let py_64 = py + .import("sys") + .unwrap() + .getattr("float_info") + .unwrap() + .getattr("max") + .unwrap(); + let rs_32 = py_64.extract::<$wrapper>().unwrap(); + // The python f64 is not representable in a rust f32 + assert!(rs_32.is_infinite()); + }) + } + }; + } + py64_rs32!(ordered_float_f32, OrderedFloat, f32); + py64_rs32!(ordered_float_f64, OrderedFloat, f64); + py64_rs32!(not_nan_f32, NotNan, f32); + py64_rs32!(not_nan_f64, NotNan, f64); +} diff --git a/src/conversions/rust_decimal.rs b/src/conversions/rust_decimal.rs index 392971a0b4b..8c5d870a2dd 100644 --- a/src/conversions/rust_decimal.rs +++ b/src/conversions/rust_decimal.rs @@ -169,8 +169,7 @@ mod test_rust_decimal { locals.set_item("rs_dec", &rs_dec).unwrap(); py.run( &CString::new(format!( - "import decimal\npy_dec = decimal.Decimal(\"{}\")\nassert py_dec == rs_dec", - num)).unwrap(), + "import decimal\npy_dec = decimal.Decimal(\"{num}\")\nassert py_dec == rs_dec")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: Decimal = rs_dec.extract().unwrap(); assert_eq!(num, roundtripped); diff --git a/src/conversions/std/array.rs b/src/conversions/std/array.rs index 36db5ec640f..1d88965e711 100644 --- a/src/conversions/std/array.rs +++ b/src/conversions/std/array.rs @@ -110,8 +110,7 @@ where fn invalid_sequence_length(expected: usize, actual: usize) -> PyErr { exceptions::PyValueError::new_err(format!( - "expected a sequence of length {} (got {})", - expected, actual + "expected a sequence of length {expected} (got {actual})" )) } diff --git a/src/conversions/std/ipaddr.rs b/src/conversions/std/ipaddr.rs index 76f6a6927c2..0c479a2d836 100644 --- a/src/conversions/std/ipaddr.rs +++ b/src/conversions/std/ipaddr.rs @@ -123,7 +123,7 @@ mod test_ipaddr { let pyobj = ip.into_pyobject(py).unwrap(); let repr = pyobj.repr().unwrap(); let repr = repr.to_string_lossy(); - assert_eq!(repr, format!("{}('{}')", py_cls, ip)); + assert_eq!(repr, format!("{py_cls}('{ip}')")); let ip2: IpAddr = pyobj.extract().unwrap(); assert_eq!(ip, ip2); diff --git a/src/conversions/std/num.rs b/src/conversions/std/num.rs index 40073d8af69..ea5798cac99 100644 --- a/src/conversions/std/num.rs +++ b/src/conversions/std/num.rs @@ -606,7 +606,7 @@ mod test_128bit_integers { let x_py = x.into_pyobject(py).unwrap(); let locals = PyDict::new(py); locals.set_item("x_py", &x_py).unwrap(); - py.run(&CString::new(format!("assert x_py == {}", x)).unwrap(), None, Some(&locals)).unwrap(); + py.run(&CString::new(format!("assert x_py == {x}")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: i128 = x_py.extract().unwrap(); assert_eq!(x, roundtripped); }) @@ -622,7 +622,7 @@ mod test_128bit_integers { let x_py = x.into_pyobject(py).unwrap(); let locals = PyDict::new(py); locals.set_item("x_py", &x_py).unwrap(); - py.run(&CString::new(format!("assert x_py == {}", x)).unwrap(), None, Some(&locals)).unwrap(); + py.run(&CString::new(format!("assert x_py == {x}")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: NonZeroI128 = x_py.extract().unwrap(); assert_eq!(x, roundtripped); }) @@ -637,7 +637,7 @@ mod test_128bit_integers { let x_py = x.into_pyobject(py).unwrap(); let locals = PyDict::new(py); locals.set_item("x_py", &x_py).unwrap(); - py.run(&CString::new(format!("assert x_py == {}", x)).unwrap(), None, Some(&locals)).unwrap(); + py.run(&CString::new(format!("assert x_py == {x}")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: u128 = x_py.extract().unwrap(); assert_eq!(x, roundtripped); }) @@ -653,7 +653,7 @@ mod test_128bit_integers { let x_py = x.into_pyobject(py).unwrap(); let locals = PyDict::new(py); locals.set_item("x_py", &x_py).unwrap(); - py.run(&CString::new(format!("assert x_py == {}", x)).unwrap(), None, Some(&locals)).unwrap(); + py.run(&CString::new(format!("assert x_py == {x}")).unwrap(), None, Some(&locals)).unwrap(); let roundtripped: NonZeroU128 = x_py.extract().unwrap(); assert_eq!(x, roundtripped); }) diff --git a/src/conversions/std/option.rs b/src/conversions/std/option.rs index cd5edec7d6d..ae0ec441c61 100644 --- a/src/conversions/std/option.rs +++ b/src/conversions/std/option.rs @@ -1,6 +1,6 @@ use crate::{ - conversion::IntoPyObject, ffi, types::any::PyAnyMethods, AsPyPointer, Bound, BoundObject, - FromPyObject, PyAny, PyResult, Python, + conversion::IntoPyObject, types::any::PyAnyMethods, Bound, BoundObject, FromPyObject, PyAny, + PyResult, Python, }; impl<'py, T> IntoPyObject<'py> for Option @@ -49,38 +49,3 @@ where } } } - -/// Convert `None` into a null pointer. -unsafe impl AsPyPointer for Option -where - T: AsPyPointer, -{ - #[inline] - fn as_ptr(&self) -> *mut ffi::PyObject { - self.as_ref() - .map_or_else(std::ptr::null_mut, |t| t.as_ptr()) - } -} - -#[cfg(test)] -mod tests { - use crate::{PyObject, Python}; - - #[test] - fn test_option_as_ptr() { - Python::with_gil(|py| { - use crate::AsPyPointer; - let mut option: Option = None; - assert_eq!(option.as_ptr(), std::ptr::null_mut()); - - let none = py.None(); - option = Some(none.clone_ref(py)); - - let ref_cnt = none.get_refcnt(py); - assert_eq!(option.as_ptr(), none.as_ptr()); - - // Ensure ref count not changed by as_ptr call - assert_eq!(none.get_refcnt(py), ref_cnt); - }); - } -} diff --git a/src/conversions/std/time.rs b/src/conversions/std/time.rs index 107ccf34cd8..53e3bf1a641 100644 --- a/src/conversions/std/time.rs +++ b/src/conversions/std/time.rs @@ -6,7 +6,7 @@ use crate::sync::GILOnceCell; use crate::types::any::PyAnyMethods; #[cfg(not(Py_LIMITED_API))] use crate::types::PyDeltaAccess; -use crate::types::{timezone_utc, PyDateTime, PyDelta}; +use crate::types::{PyDateTime, PyDelta, PyTzInfo}; use crate::{Borrowed, Bound, FromPyObject, Py, PyAny, PyErr, PyResult, Python}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -128,9 +128,8 @@ fn unix_epoch_py(py: Python<'_>) -> PyResult> { static UNIX_EPOCH: GILOnceCell> = GILOnceCell::new(); Ok(UNIX_EPOCH .get_or_try_init(py, || { - Ok::<_, PyErr>( - PyDateTime::new(py, 1970, 1, 1, 0, 0, 0, 0, Some(&timezone_utc(py)))?.into(), - ) + let utc = PyTzInfo::utc(py)?; + Ok::<_, PyErr>(PyDateTime::new(py, 1970, 1, 1, 0, 0, 0, 0, Some(&utc))?.into()) })? .bind_borrowed(py)) } @@ -138,7 +137,7 @@ fn unix_epoch_py(py: Python<'_>) -> PyResult> { #[cfg(test)] mod tests { use super::*; - use crate::types::{timezone_utc, PyDict}; + use crate::types::PyDict; #[test] fn test_duration_frompyobject() { @@ -320,6 +319,7 @@ mod tests { second: u8, microsecond: u32, ) -> Bound<'_, PyDateTime> { + let utc = PyTzInfo::utc(py).unwrap(); PyDateTime::new( py, year, @@ -329,7 +329,7 @@ mod tests { minute, second, microsecond, - Some(&timezone_utc(py)), + Some(&utc), ) .unwrap() } @@ -337,7 +337,9 @@ mod tests { fn max_datetime(py: Python<'_>) -> Bound<'_, PyDateTime> { let naive_max = datetime_class(py).getattr("max").unwrap(); let kargs = PyDict::new(py); - kargs.set_item("tzinfo", timezone_utc(py)).unwrap(); + kargs + .set_item("tzinfo", PyTzInfo::utc(py).unwrap()) + .unwrap(); naive_max .call_method("replace", (), Some(&kargs)) .unwrap() diff --git a/src/conversions/time.rs b/src/conversions/time.rs new file mode 100644 index 00000000000..f077b3be55d --- /dev/null +++ b/src/conversions/time.rs @@ -0,0 +1,1504 @@ +#![cfg(feature = "time")] + +//! Conversions to and from [time](https://docs.rs/time/)’s `Date`, +//! `Duration`, `OffsetDateTime`, `PrimitiveDateTime`, `Time`, `UtcDateTime` and `UtcOffset`. +//! +//! # Setup +//! +//! To use this feature, add this to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +//! time = "0.3" +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"time\"] }")] +//! ``` +//! +//! Note that you must use compatible versions of time and PyO3. +//! The required time version may vary based on the version of PyO3. +//! +//! ```rust +//! use time::{Duration, OffsetDateTime, PrimitiveDateTime, Date, Time, Month}; +//! use pyo3::{Python, PyResult, IntoPyObject, types::PyAnyMethods}; +//! +//! fn main() -> PyResult<()> { +//! pyo3::prepare_freethreaded_python(); +//! Python::with_gil(|py| { +//! // Create a fixed date and time (2022-01-01 12:00:00 UTC) +//! let date = Date::from_calendar_date(2022, Month::January, 1).unwrap(); +//! let time = Time::from_hms(12, 0, 0).unwrap(); +//! let primitive_dt = PrimitiveDateTime::new(date, time); +//! +//! // Convert to OffsetDateTime with UTC offset +//! let datetime = primitive_dt.assume_utc(); +//! +//! // Create a duration of 1 hour +//! let duration = Duration::hours(1); +//! +//! // Convert to Python objects +//! let py_datetime = datetime.into_pyobject(py)?; +//! let py_timedelta = duration.into_pyobject(py)?; +//! +//! // Add the duration to the datetime in Python +//! let py_result = py_datetime.add(py_timedelta)?; +//! +//! // Convert the result back to Rust +//! let result: OffsetDateTime = py_result.extract()?; +//! assert_eq!(result.hour(), 13); +//! +//! Ok(()) +//! }) +//! } +//! ``` + +use crate::exceptions::{PyTypeError, PyValueError}; +#[cfg(Py_LIMITED_API)] +use crate::intern; +#[cfg(not(Py_LIMITED_API))] +use crate::types::datetime::{PyDateAccess, PyDeltaAccess}; +use crate::types::{PyAnyMethods, PyDate, PyDateTime, PyDelta, PyNone, PyTime, PyTzInfo}; +#[cfg(not(Py_LIMITED_API))] +use crate::types::{PyTimeAccess, PyTzInfoAccess}; +use crate::{Bound, FromPyObject, IntoPyObject, PyAny, PyErr, PyResult, Python}; +use time::{ + Date, Duration, Month, OffsetDateTime, PrimitiveDateTime, Time, UtcDateTime, UtcOffset, +}; + +const SECONDS_PER_DAY: i64 = 86_400; + +// Macro for reference implementation +macro_rules! impl_into_py_for_ref { + ($type:ty, $target:ty) => { + impl<'py> IntoPyObject<'py> for &$type { + type Target = $target; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + #[inline] + fn into_pyobject(self, py: Python<'py>) -> Result { + (*self).into_pyobject(py) + } + } + }; +} + +// Macro for month conversion +macro_rules! month_from_number { + ($month:expr) => { + match $month { + 1 => Month::January, + 2 => Month::February, + 3 => Month::March, + 4 => Month::April, + 5 => Month::May, + 6 => Month::June, + 7 => Month::July, + 8 => Month::August, + 9 => Month::September, + 10 => Month::October, + 11 => Month::November, + 12 => Month::December, + _ => return Err(PyValueError::new_err("invalid month value")), + } + }; +} + +fn extract_date_time(dt: &Bound<'_, PyAny>) -> PyResult<(Date, Time)> { + #[cfg(not(Py_LIMITED_API))] + { + let dt = dt.downcast::()?; + let date = Date::from_calendar_date( + dt.get_year(), + month_from_number!(dt.get_month()), + dt.get_day(), + ) + .map_err(|_| PyValueError::new_err("invalid or out-of-range date"))?; + + let time = Time::from_hms_micro( + dt.get_hour(), + dt.get_minute(), + dt.get_second(), + dt.get_microsecond(), + ) + .map_err(|_| PyValueError::new_err("invalid or out-of-range time"))?; + Ok((date, time)) + } + + #[cfg(Py_LIMITED_API)] + { + let date = Date::from_calendar_date( + dt.getattr(intern!(dt.py(), "year"))?.extract()?, + month_from_number!(dt.getattr(intern!(dt.py(), "month"))?.extract::()?), + dt.getattr(intern!(dt.py(), "day"))?.extract()?, + ) + .map_err(|_| PyValueError::new_err("invalid or out-of-range date"))?; + + let time = Time::from_hms_micro( + dt.getattr(intern!(dt.py(), "hour"))?.extract()?, + dt.getattr(intern!(dt.py(), "minute"))?.extract()?, + dt.getattr(intern!(dt.py(), "second"))?.extract()?, + dt.getattr(intern!(dt.py(), "microsecond"))?.extract()?, + ) + .map_err(|_| PyValueError::new_err("invalid or out-of-range time"))?; + + Ok((date, time)) + } +} + +impl<'py> IntoPyObject<'py> for Duration { + type Target = PyDelta; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let total_seconds = self.whole_seconds(); + let micro_seconds = self.subsec_microseconds(); + + // For negative durations, Python expects days to be negative and + // seconds/microseconds to be positive or zero + let (days, seconds) = if total_seconds < 0 && total_seconds % SECONDS_PER_DAY != 0 { + // For negative values, we need to round down (toward more negative) + // e.g., -10 seconds should be -1 days + 86390 seconds + let days = total_seconds.div_euclid(SECONDS_PER_DAY); + let seconds = total_seconds.rem_euclid(SECONDS_PER_DAY); + (days, seconds) + } else { + // For positive or exact negative days, use normal division + ( + total_seconds / SECONDS_PER_DAY, + total_seconds % SECONDS_PER_DAY, + ) + }; + // Create the timedelta with days, seconds, microseconds + // Safe to unwrap as we've verified the values are within bounds + PyDelta::new( + py, + days.try_into().expect("days overflow"), + seconds.try_into().expect("seconds overflow"), + micro_seconds, + true, + ) + } +} + +impl FromPyObject<'_> for Duration { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { + #[cfg(not(Py_LIMITED_API))] + let (days, seconds, microseconds) = { + let delta = ob.downcast::()?; + ( + delta.get_days().into(), + delta.get_seconds().into(), + delta.get_microseconds().into(), + ) + }; + + #[cfg(Py_LIMITED_API)] + let (days, seconds, microseconds) = { + ( + ob.getattr(intern!(ob.py(), "days"))?.extract()?, + ob.getattr(intern!(ob.py(), "seconds"))?.extract()?, + ob.getattr(intern!(ob.py(), "microseconds"))?.extract()?, + ) + }; + + Ok( + Duration::days(days) + + Duration::seconds(seconds) + + Duration::microseconds(microseconds), + ) + } +} + +impl<'py> IntoPyObject<'py> for Date { + type Target = PyDate; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let year = self.year(); + let month = self.month() as u8; + let day = self.day(); + + PyDate::new(py, year, month, day) + } +} + +impl FromPyObject<'_> for Date { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { + let (year, month, day) = { + #[cfg(not(Py_LIMITED_API))] + { + let date = ob.downcast::()?; + (date.get_year(), date.get_month(), date.get_day()) + } + + #[cfg(Py_LIMITED_API)] + { + let year = ob.getattr(intern!(ob.py(), "year"))?.extract()?; + let month: u8 = ob.getattr(intern!(ob.py(), "month"))?.extract()?; + let day = ob.getattr(intern!(ob.py(), "day"))?.extract()?; + (year, month, day) + } + }; + + // Convert the month number to time::Month enum + let month = month_from_number!(month); + + Date::from_calendar_date(year, month, day) + .map_err(|_| PyValueError::new_err("invalid or out-of-range date")) + } +} + +impl<'py> IntoPyObject<'py> for Time { + type Target = PyTime; + type Output = Bound<'py, Self::Target>; + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let hour = self.hour(); + let minute = self.minute(); + let second = self.second(); + let microsecond = self.microsecond(); + + PyTime::new(py, hour, minute, second, microsecond, None) + } +} + +impl FromPyObject<'_> for Time { + fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult