diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5042ac9cee3..76c5ab06f29 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,7 +115,7 @@ jobs: { os: "ubuntu-latest", python-architecture: "x64", - rust-target: "wasm32-wasi", + rust-target: "wasm32-wasip1", }, { os: "windows-latest", @@ -489,7 +489,7 @@ jobs: components: rust-src - uses: actions/setup-node@v4 with: - node-version: 14 + node-version: 18 - run: python -m pip install --upgrade pip && pip install nox - uses: actions/cache@v4 id: cache @@ -656,14 +656,17 @@ jobs: # ubuntu x86_64 -> windows x86_64 - os: "ubuntu-latest" target: "x86_64-pc-windows-gnu" - flags: "-i python3.12 --features abi3 --features generate-import-lib" - manylinux: off + flags: "-i python3.12 --features generate-import-lib" # macos x86_64 -> aarch64 - os: "macos-13" # last x86_64 macos runners target: "aarch64-apple-darwin" # macos aarch64 -> x86_64 - os: "macos-latest" target: "x86_64-apple-darwin" + # windows x86_64 -> aarch64 + - os: "windows-latest" + target: "aarch64-pc-windows-msvc" + flags: "-i python3.12 --features generate-import-lib" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -677,11 +680,18 @@ jobs: - name: Setup cross-compiler if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} run: sudo apt-get install -y mingw-w64 llvm - - uses: PyO3/maturin-action@v1 + - name: Compile version-specific library + uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux }} args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }} + - name: Compile abi3 library + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }} test-cross-compilation-windows: needs: [fmt] diff --git a/.github/workflows/coverage-pr-base.yml b/.github/workflows/coverage-pr-base.yml index acc645ea876..33118697636 100644 --- a/.github/workflows/coverage-pr-base.yml +++ b/.github/workflows/coverage-pr-base.yml @@ -19,9 +19,6 @@ jobs: - name: Set PR base on codecov run: | # fetch the merge commit between the PR base and head - BASE_REF=refs/heads/${{ github.event.pull_request.base.ref }} - MERGE_REF=refs/pull/${{ github.event.pull_request.number }}/merge - git fetch -u --progress --depth=1 origin "+$BASE_REF:$BASE_REF" "+$MERGE_REF:$MERGE_REF" while [ -z "$(git merge-base "$BASE_REF" "$MERGE_REF")" ]; do git fetch -u -q --deepen="10" origin "$BASE_REF" "$MERGE_REF"; @@ -38,3 +35,8 @@ jobs: --slug PyO3/pyo3 \ --token ${{ secrets.CODECOV_TOKEN }} \ --service github + env: + # Don't put these in bash, because we don't want the expansion to + # risk code execution + BASE_REF: "refs/heads/${{ github.event.pull_request.base.ref }}" + MERGE_REF: "refs/pull/${{ github.event.pull_request.number }}/merge" diff --git a/CHANGELOG.md b/CHANGELOG.md index 668fbd23d43..dfcf20e6fa3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,34 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.23.4] - 2025-01-10 + +### Added + +- Add `PyList::locked_for_each`, which uses a critical section to lock the list on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Add `pyo3_build_config::add_python_framework_link_args` build script API to set rpath when using macOS system Python. [#4833](https://github.com/PyO3/pyo3/pull/4833) + +### Changed + +- Use `datetime.fold` to distinguish ambiguous datetimes when converting to and from `chrono::DateTime` (rather than erroring). [#4791](https://github.com/PyO3/pyo3/pull/4791) +- Optimize PyList iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) + +### Fixed + +- Fix unnecessary internal `py.allow_threads` GIL-switch when attempting to access contents of a `PyErr` which originated from Python (could lead to unintended deadlocks). [#4766](https://github.com/PyO3/pyo3/pull/4766) +- Fix thread-unsafe access of dict internals in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +* Fix unnecessary critical sections in `BoundDictIterator` on the free-threaded build. [#4788](https://github.com/PyO3/pyo3/pull/4788) +- Fix time-of-check to time-of-use issues with list iteration on the free-threaded build. [#4789](https://github.com/PyO3/pyo3/pull/4789) +- Fix `chrono::DateTime` to-Python conversion when `Tz` is `chrono_tz::Tz`. [#4790](https://github.com/PyO3/pyo3/pull/4790) +- Fix `#[pyclass]` not being able to be named `Probe`. [#4794](https://github.com/PyO3/pyo3/pull/4794) +- Fix not treating cross-compilation from x64 to aarch64 on Windows as a cross-compile. [#4800](https://github.com/PyO3/pyo3/pull/4800) +- Fix missing struct fields on GraalPy when subclassing builtin classes. [#4802](https://github.com/PyO3/pyo3/pull/4802) +- Fix generating import lib for PyPy when `abi3` feature is enabled. [#4806](https://github.com/PyO3/pyo3/pull/4806) +- Fix generating import lib for python3.13t when `abi3` feature is enabled. [#4808](https://github.com/PyO3/pyo3/pull/4808) +- Fix compile failure for raw identifiers like `r#box` in `derive(FromPyObject)`. [#4814](https://github.com/PyO3/pyo3/pull/4814) +- Fix compile failure for `#[pyclass]` enum variants with more than 12 fields. [#4832](https://github.com/PyO3/pyo3/pull/4832) + + ## [0.23.3] - 2024-12-03 ### Packaging @@ -2026,7 +2054,8 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.23.3...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.23.4...HEAD +[0.23.4]: https://github.com/pyo3/pyo3/compare/v0.23.3...v0.23.4 [0.23.3]: https://github.com/pyo3/pyo3/compare/v0.23.2...v0.23.3 [0.23.2]: https://github.com/pyo3/pyo3/compare/v0.23.1...v0.23.2 [0.23.1]: https://github.com/pyo3/pyo3/compare/v0.23.0...v0.23.1 diff --git a/Cargo.toml b/Cargo.toml index 18cceffbd0c..22c18228c44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.23.3" +version = "0.23.4" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -21,10 +21,10 @@ memoffset = "0.9" once_cell = "1.13" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.23.3" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.23.4" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.23.3", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.23.4", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -66,7 +66,7 @@ static_assertions = "1.1.0" uuid = {version = "1.10.0", features = ["v4"] } [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.23.3", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.23.4", features = ["resolve-config"] } [features] default = ["macros"] diff --git a/README.md b/README.md index 82b8d390981..ece3524816f 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.23.3", features = ["extension-module"] } +pyo3 = { version = "0.23.4", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -140,7 +140,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.23.3" +version = "0.23.4" features = ["auto-initialize"] ``` diff --git a/build.rs b/build.rs index 5d638291f3b..68a658bf285 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,9 @@ use std::env; use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; -use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; +use pyo3_build_config::{ + add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, +}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared { @@ -42,6 +44,9 @@ fn configure_pyo3() -> Result<()> { // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); + // Make `cargo test` etc work on macOS with Xcode bundled Python + add_python_framework_link_args(); + Ok(()) } diff --git a/emscripten/Makefile b/emscripten/Makefile index af224854c26..54094382c1b 100644 --- a/emscripten/Makefile +++ b/emscripten/Makefile @@ -4,7 +4,7 @@ CURDIR=$(abspath .) BUILDROOT ?= $(CURDIR)/builddir PYMAJORMINORMICRO ?= 3.11.0 -EMSCRIPTEN_VERSION=3.1.13 +EMSCRIPTEN_VERSION=3.1.68 export EMSDKDIR = $(BUILDROOT)/emsdk diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index aa83bdd81b7..a4e89a0d054 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.23.3"); +variable::set("PYO3_VERSION", "0.23.4"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index aa83bdd81b7..a4e89a0d054 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.23.3"); +variable::set("PYO3_VERSION", "0.23.4"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index dd76eff3b7b..3a1392912ca 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.23.3"); +variable::set("PYO3_VERSION", "0.23.4"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index 461282c2832..6b93bf9e6d1 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.23.3"); +variable::set("PYO3_VERSION", "0.23.4"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index aa83bdd81b7..a4e89a0d054 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.23.3"); +variable::set("PYO3_VERSION", "0.23.4"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 1a806304d22..d3474fedaf7 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -144,7 +144,17 @@ rustflags = [ ] ``` -Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. These can be resolved with another addition to `.cargo/config.toml`: +Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. + +The easiest way to set the correct linker arguments is to add a `build.rs` with the following content: + +```rust,ignore +fn main() { + pyo3_build_config::add_python_framework_link_args(); +} +``` + +Alternatively it can be resolved with another addition to `.cargo/config.toml`: ```toml [build] @@ -153,16 +163,6 @@ rustflags = [ ] ``` -Alternatively, one can include in `build.rs`: - -```rust -fn main() { - println!( - "cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks" - ); -} -``` - For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649). Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md index f212cb0b9a9..8ee9a2e100e 100644 --- a/guide/src/free-threading.md +++ b/guide/src/free-threading.md @@ -156,20 +156,40 @@ freethreaded build, holding a `'py` lifetime means only that the thread is currently attached to the Python interpreter -- other threads can be simultaneously interacting with the interpreter. -The main reason for obtaining a `'py` lifetime is to interact with Python +You still need to obtain a `'py` lifetime is to interact with Python objects or call into the CPython C API. If you are not yet attached to the Python runtime, you can register a thread using the [`Python::with_gil`] function. Threads created via the Python [`threading`] module do not not need to -do this, but all other OS threads that interact with the Python runtime must -explicitly attach using `with_gil` and obtain a `'py` liftime. - -Since there is no GIL in the free-threaded build, releasing the GIL for -long-running tasks is no longer necessary to ensure other threads run, but you -should still detach from the interpreter runtime using [`Python::allow_threads`] -when doing long-running tasks that do not require the CPython runtime. The -garbage collector can only run if all threads are detached from the runtime (in -a stop-the-world state), so detaching from the runtime allows freeing unused -memory. +do this, and pyo3 will handle setting up the [`Python<'py>`] token when CPython +calls into your extension. + +### Global synchronization events can cause hangs and deadlocks + +The free-threaded build triggers global synchronization events in the following +situations: + +* During garbage collection in order to get a globally consistent view of + reference counts and references between objects +* In Python 3.13, when the first background thread is started in + order to mark certain objects as immortal +* When either `sys.settrace` or `sys.setprofile` are called in order to + instrument running code objects and threads +* Before `os.fork()` is called. + +This is a non-exhaustive list and there may be other situations in future Python +versions that can trigger global synchronization events. + +This means that you should detach from the interpreter runtime using +[`Python::allow_threads`] in exactly the same situations as you should detach +from the runtime in the GIL-enabled build: when doing long-running tasks that do +not require the CPython runtime or when doing any task that needs to re-attach +to the runtime (see the [guide +section](parallelism.md#sharing-python-objects-between-rust-threads) that +covers this). In the former case, you would observe a hang on threads that are +waiting on the long-running task to complete, and in the latter case you would +see a deadlock while a thread tries to attach after the runtime triggers a +global synchronization event, but the spawning thread prevents the +synchronization event from completing. ### Exceptions and panics for multithreaded access of mutable `pyclass` instances @@ -183,10 +203,10 @@ mutability](./class.md#bound-and-interior-mutability),) but now in free-threaded Python there are more opportunities to trigger these panics from Python because there is no GIL to lock concurrent access to mutably borrowed data from Python. -The most straightforward way to trigger this problem to use the Python +The most straightforward way to trigger this problem is to use the Python [`threading`] module to simultaneously call a rust function that mutably borrows a -[`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html). For example, -consider the following implementation: +[`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html) in multiple threads. For +example, consider the following implementation: ```rust # use pyo3::prelude::*; @@ -240,7 +260,7 @@ RuntimeError: Already borrowed We plan to allow user-selectable semantics for mutable pyclass definitions in PyO3 0.24, allowing some form of opt-in locking to emulate the GIL if that is needed. For now you should explicitly add locking, possibly using conditional -compilation or using the critical section API to avoid creating deadlocks with +compilation or using the critical section API, to avoid creating deadlocks with the GIL. ### Cannot build extensions using the limited API @@ -252,8 +272,8 @@ PyO3 will print a warning and ignore that setting when building extensions using the free-threaded interpreter. This means that if your package makes use of the ABI forward compatibility -provided by the limited API to uploads only one wheel for each release of your -package, you will need to update and tooling or instructions to also upload a +provided by the limited API to upload only one wheel for each release of your +package, you will need to update your release procedure to also upload a version-specific free-threaded wheel. See [the guide section](./building-and-distribution/multiple-python-versions.md) diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index a288b14be19..64ff1c8c9c0 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -1,6 +1,6 @@ # Parallelism -CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. +CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) (GIL), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. There is an experimental "free-threaded" version of CPython 3.13 that does not have a GIL, see the PyO3 docs on [free-threaded Python](./free-threading.md) for more information about that. In PyO3 parallelism can be easily achieved in Rust-only code. Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [rayon](https://github.com/rayon-rs/rayon) crate to count words in parallel. ```rust,no_run @@ -117,4 +117,61 @@ test_word_count_python_sequential 27.3985 (15.82) 45.452 You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled. +## Sharing Python objects between Rust threads + +In the example above we made a Python interface to a low-level rust function, +and then leveraged the python `threading` module to run the low-level function +in parallel. It is also possible to spawn threads in Rust that acquire the GIL +and operate on Python objects. However, care must be taken to avoid writing code +that deadlocks with the GIL in these cases. + +* Note: This example is meant to illustrate how to drop and re-acquire the GIL + to avoid creating deadlocks. Unless the spawned threads subsequently + release the GIL or you are using the free-threaded build of CPython, you + will not see any speedups due to multi-threaded parallelism using `rayon` + to parallelize code that acquires and holds the GIL for the entire + execution of the spawned thread. + +In the example below, we share a `Vec` of User ID objects defined using the +`pyclass` macro and spawn threads to process the collection of data into a `Vec` +of booleans based on a predicate using a rayon parallel iterator: + +```rust,no_run +use pyo3::prelude::*; + +// These traits let us use int_par_iter and map +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +#[pyclass] +struct UserID { + id: i64, +} + +let allowed_ids: Vec = Python::with_gil(|outer_py| { + let instances: Vec> = (0..10).map(|x| Py::new(outer_py, UserID { id: x }).unwrap()).collect(); + outer_py.allow_threads(|| { + instances.par_iter().map(|instance| { + Python::with_gil(|inner_py| { + instance.borrow(inner_py).id > 5 + }) + }).collect() + }) +}); +assert!(allowed_ids.into_iter().filter(|b| *b).count() == 4); +``` + +It's important to note that there is an `outer_py` GIL lifetime token as well as +an `inner_py` token. Sharing GIL lifetime tokens between threads is not allowed +and threads must individually acquire the GIL to access data wrapped by a python +object. + +It's also important to see that this example uses [`Python::allow_threads`] to +wrap the code that spawns OS threads via `rayon`. If this example didn't use +`allow_threads`, a rayon worker thread would block on acquiring the GIL while a +thread that owns the GIL spins forever waiting for the result of the rayon +thread. Calling `allow_threads` allows the GIL to be released in the thread +collecting the results from the worker threads. You should always call +`allow_threads` in situations that spawn worker threads, but especially so in +cases where worker threads need to acquire the GIL, to prevent deadlocks. + [`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.allow_threads diff --git a/noxfile.py b/noxfile.py index 25cb8d1eb92..ed7759f5f99 100644 --- a/noxfile.py +++ b/noxfile.py @@ -339,6 +339,7 @@ def test_emscripten(session: nox.Session): f"-C link-arg=-lpython{info.pymajorminor}", "-C link-arg=-lexpat", "-C link-arg=-lmpdec", + "-C link-arg=-lsqlite3", "-C link-arg=-lz", "-C link-arg=-lbz2", "-C link-arg=-sALLOW_MEMORY_GROWTH=1", @@ -351,7 +352,7 @@ def test_emscripten(session: nox.Session): session, "bash", "-c", - f"source {info.builddir/'emsdk/emsdk_env.sh'} && cargo test", + f"source {info.builddir / 'emsdk/emsdk_env.sh'} && cargo test", ) @@ -797,7 +798,7 @@ def _get_rust_default_target() -> str: def _get_feature_sets() -> Tuple[Tuple[str, ...], ...]: """Returns feature sets to use for clippy job""" cargo_target = os.getenv("CARGO_BUILD_TARGET", "") - if "wasm32-wasi" not in cargo_target: + if "wasm32-wasip1" not in cargo_target: # multiple-pymethods not supported on wasm return ( ("--no-default-features",), @@ -951,7 +952,7 @@ def set( f"""\ implementation={implementation} version={version} -build_flags={','.join(build_flags)} +build_flags={",".join(build_flags)} suppress_build_script_link_lines=true """ ) diff --git a/pyo3-benches/benches/bench_call.rs b/pyo3-benches/benches/bench_call.rs index bbcf3ac2bdf..b6e090a7dbd 100644 --- a/pyo3-benches/benches/bench_call.rs +++ b/pyo3-benches/benches/bench_call.rs @@ -33,9 +33,9 @@ fn bench_call_1(b: &mut Bencher<'_>) { let foo_module = &module.getattr("foo").unwrap(); let args = ( - <_ as IntoPy>::into_py(1, py).into_bound(py), - <_ as IntoPy>::into_py("s", py).into_bound(py), - <_ as IntoPy>::into_py(1.23, py).into_bound(py), + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), ); b.iter(|| { @@ -52,9 +52,9 @@ fn bench_call(b: &mut Bencher<'_>) { let foo_module = &module.getattr("foo").unwrap(); let args = ( - <_ as IntoPy>::into_py(1, py).into_bound(py), - <_ as IntoPy>::into_py("s", py).into_bound(py), - <_ as IntoPy>::into_py(1.23, py).into_bound(py), + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), ); let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); @@ -73,7 +73,7 @@ fn bench_call_one_arg(b: &mut Bencher<'_>) { let module = test_module!(py, "def foo(a): pass"); let foo_module = &module.getattr("foo").unwrap(); - let arg = <_ as IntoPy>::into_py(1, py).into_bound(py); + let arg = 1i32.into_pyobject(py).unwrap(); b.iter(|| { for _ in 0..1000 { @@ -117,9 +117,9 @@ class Foo: let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); let args = ( - <_ as IntoPy>::into_py(1, py).into_bound(py), - <_ as IntoPy>::into_py("s", py).into_bound(py), - <_ as IntoPy>::into_py(1.23, py).into_bound(py), + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), ); b.iter(|| { @@ -145,9 +145,9 @@ class Foo: let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); let args = ( - <_ as IntoPy>::into_py(1, py).into_bound(py), - <_ as IntoPy>::into_py("s", py).into_bound(py), - <_ as IntoPy>::into_py(1.23, py).into_bound(py), + 1.into_pyobject(py).unwrap(), + "s".into_pyobject(py).unwrap(), + 1.23.into_pyobject(py).unwrap(), ); let kwargs = [("d", 1), ("e", 42)].into_py_dict(py).unwrap(); @@ -173,7 +173,7 @@ class Foo: ); let foo_module = &module.getattr("Foo").unwrap().call0().unwrap(); - let arg = <_ as IntoPy>::into_py(1, py).into_bound(py); + let arg = 1i32.into_pyobject(py).unwrap(); b.iter(|| { for _ in 0..1000 { diff --git a/pyo3-benches/benches/bench_extract.rs b/pyo3-benches/benches/bench_extract.rs index a261b14cfa1..2062ccba7b5 100644 --- a/pyo3-benches/benches/bench_extract.rs +++ b/pyo3-benches/benches/bench_extract.rs @@ -26,7 +26,7 @@ fn extract_str_extract_fail(bench: &mut Bencher<'_>) { }); } -#[cfg(any(Py_3_10))] +#[cfg(Py_3_10)] fn extract_str_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { let s = PyString::new(py, "Hello, World!").into_any(); @@ -51,7 +51,7 @@ fn extract_str_downcast_fail(bench: &mut Bencher<'_>) { fn extract_int_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = 123.to_object(py).into_bound(py); + let int = 123i32.into_pyobject(py).unwrap(); bench.iter(|| black_box(&int).extract::().unwrap()); }); @@ -70,7 +70,7 @@ fn extract_int_extract_fail(bench: &mut Bencher<'_>) { fn extract_int_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let int = 123.to_object(py).into_bound(py); + let int = 123i32.into_pyobject(py).unwrap(); bench.iter(|| { let py_int = black_box(&int).downcast::().unwrap(); @@ -92,7 +92,7 @@ fn extract_int_downcast_fail(bench: &mut Bencher<'_>) { fn extract_float_extract_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float = 23.42.to_object(py).into_bound(py); + let float = 23.42f64.into_pyobject(py).unwrap(); bench.iter(|| black_box(&float).extract::().unwrap()); }); @@ -111,7 +111,7 @@ fn extract_float_extract_fail(bench: &mut Bencher<'_>) { fn extract_float_downcast_success(bench: &mut Bencher<'_>) { Python::with_gil(|py| { - let float = 23.42.to_object(py).into_bound(py); + let float = 23.42f64.into_pyobject(py).unwrap(); bench.iter(|| { let py_float = black_box(&float).downcast::().unwrap(); diff --git a/pyo3-benches/benches/bench_intopyobject.rs b/pyo3-benches/benches/bench_intopyobject.rs index 16351c9088b..42af893cd8a 100644 --- a/pyo3-benches/benches/bench_intopyobject.rs +++ b/pyo3-benches/benches/bench_intopyobject.rs @@ -17,7 +17,7 @@ fn bytes_new_small(b: &mut Bencher<'_>) { } fn bytes_new_medium(b: &mut Bencher<'_>) { - let data = (0..u8::MAX).into_iter().collect::>(); + let data = (0..u8::MAX).collect::>(); bench_bytes_new(b, &data); } @@ -37,7 +37,7 @@ fn byte_slice_into_pyobject_small(b: &mut Bencher<'_>) { } fn byte_slice_into_pyobject_medium(b: &mut Bencher<'_>) { - let data = (0..u8::MAX).into_iter().collect::>(); + let data = (0..u8::MAX).collect::>(); bench_bytes_into_pyobject(b, &data); } @@ -46,9 +46,10 @@ fn byte_slice_into_pyobject_large(b: &mut Bencher<'_>) { bench_bytes_into_pyobject(b, &data); } +#[allow(deprecated)] fn byte_slice_into_py(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let data = (0..u8::MAX).into_iter().collect::>(); + let data = (0..u8::MAX).collect::>(); let bytes = data.as_slice(); b.iter_with_large_drop(|| black_box(bytes).into_py(py)); }); @@ -56,14 +57,15 @@ fn byte_slice_into_py(b: &mut Bencher<'_>) { fn vec_into_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let bytes = (0..u8::MAX).into_iter().collect::>(); + let bytes = (0..u8::MAX).collect::>(); b.iter_with_large_drop(|| black_box(&bytes).clone().into_pyobject(py)); }); } +#[allow(deprecated)] fn vec_into_py(b: &mut Bencher<'_>) { Python::with_gil(|py| { - let bytes = (0..u8::MAX).into_iter().collect::>(); + let bytes = (0..u8::MAX).collect::>(); b.iter_with_large_drop(|| black_box(&bytes).clone().into_py(py)); }); } diff --git a/pyo3-benches/benches/bench_pyobject.rs b/pyo3-benches/benches/bench_pyobject.rs index a57a98a8d30..c731216c79f 100644 --- a/pyo3-benches/benches/bench_pyobject.rs +++ b/pyo3-benches/benches/bench_pyobject.rs @@ -22,13 +22,7 @@ fn drop_many_objects(b: &mut Bencher<'_>) { fn drop_many_objects_without_gil(b: &mut Bencher<'_>) { b.iter_batched( - || { - Python::with_gil(|py| { - (0..1000) - .map(|_| py.None().into_py(py)) - .collect::>() - }) - }, + || Python::with_gil(|py| (0..1000).map(|_| py.None()).collect::>()), |objs| { drop(objs); @@ -76,7 +70,7 @@ fn drop_many_objects_multiple_threads(b: &mut Bencher<'_>) { for sender in &sender { let objs = Python::with_gil(|py| { (0..1000 / THREADS) - .map(|_| py.None().into_py(py)) + .map(|_| py.None()) .collect::>() }); diff --git a/pyo3-benches/benches/bench_set.rs b/pyo3-benches/benches/bench_set.rs index 0be06309595..2d468740ea0 100644 --- a/pyo3-benches/benches/bench_set.rs +++ b/pyo3-benches/benches/bench_set.rs @@ -1,7 +1,7 @@ use codspeed_criterion_compat::{criterion_group, criterion_main, Bencher, Criterion}; -use pyo3::prelude::*; use pyo3::types::PySet; +use pyo3::{prelude::*, IntoPyObjectExt}; use std::{ collections::{BTreeSet, HashSet}, hint::black_box, @@ -12,7 +12,7 @@ fn set_new(b: &mut Bencher<'_>) { const LEN: usize = 100_000; // Create Python objects up-front, so that the benchmark doesn't need to include // the cost of allocating LEN Python integers - let elements: Vec = (0..LEN).map(|i| i.into_py(py)).collect(); + let elements: Vec = (0..LEN).map(|i| i.into_py_any(py).unwrap()).collect(); b.iter_with_large_drop(|| PySet::new(py, &elements).unwrap()); }); } @@ -20,7 +20,7 @@ fn set_new(b: &mut Bencher<'_>) { fn iter_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let set = PySet::new(py, &(0..LEN).collect::>()).unwrap(); + let set = PySet::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for x in &set { @@ -34,9 +34,7 @@ fn iter_set(b: &mut Bencher<'_>) { fn extract_hashset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -44,9 +42,7 @@ fn extract_hashset(b: &mut Bencher<'_>) { fn extract_btreeset(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } @@ -54,9 +50,7 @@ fn extract_btreeset(b: &mut Bencher<'_>) { fn extract_hashbrown_set(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 100_000; - let any = PySet::new(py, &(0..LEN).collect::>()) - .unwrap() - .into_any(); + let any = PySet::new(py, 0..LEN).unwrap().into_any(); b.iter_with_large_drop(|| black_box(&any).extract::>()); }); } diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index a5e43d6ef43..72146c80b54 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -115,12 +115,23 @@ fn tuple_to_list(b: &mut Bencher<'_>) { }); } +#[allow(deprecated)] fn tuple_into_py(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| -> PyObject { (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12).into_py(py) }); }); } +fn tuple_into_pyobject(b: &mut Bencher<'_>) { + Python::with_gil(|py| { + b.iter(|| { + (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) + .into_pyobject(py) + .unwrap() + }); + }); +} + fn criterion_benchmark(c: &mut Criterion) { c.bench_function("iter_tuple", iter_tuple); c.bench_function("tuple_new", tuple_new); @@ -137,6 +148,7 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); c.bench_function("tuple_into_py", tuple_into_py); + c.bench_function("tuple_into_pyobject", tuple_into_pyobject); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 1e951b29bff..cee4c3ad52b 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.23.3" +version = "0.23.4" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -13,11 +13,11 @@ rust-version = "1.63" [dependencies] once_cell = "1" -python3-dll-a = { version = "0.2.11", optional = true } +python3-dll-a = { version = "0.2.12", optional = true } target-lexicon = "0.12.14" [build-dependencies] -python3-dll-a = { version = "0.2.11", optional = true } +python3-dll-a = { version = "0.2.12", optional = true } target-lexicon = "0.12.14" [features] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index f36c5e50226..1de37da0101 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -22,7 +22,7 @@ use std::{ pub use target_lexicon::Triple; -use target_lexicon::{Environment, OperatingSystem}; +use target_lexicon::{Architecture, Environment, OperatingSystem}; use crate::{ bail, ensure, @@ -167,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -245,6 +247,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -253,6 +256,7 @@ print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -289,6 +293,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) }; let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -359,6 +364,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -396,6 +402,9 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { Some(value) => value == "1", @@ -424,6 +433,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -497,9 +507,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let mut lib_dir = None; let mut executable = None; let mut pointer_width = None; - let mut build_flags = None; + let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -528,6 +539,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -535,10 +547,12 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let version = version.ok_or("missing value for version")?; let implementation = implementation.unwrap_or(PythonImplementation::CPython); let abi3 = abi3.unwrap_or(false); + let build_flags = build_flags.unwrap_or_default(); + let gil_disabled = build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED); // Fixup lib_name if it's not set let lib_name = lib_name.or_else(|| { if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::()) { - default_lib_name_for_target(version, implementation, abi3, &target) + default_lib_name_for_target(version, implementation, abi3, gil_disabled, &target) } else { None } @@ -553,9 +567,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) lib_dir, executable, pointer_width, - build_flags: build_flags.unwrap_or_default(), + build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } @@ -565,7 +580,14 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) // Auto generate python3.dll import libraries for Windows targets. if self.lib_dir.is_none() { let target = target_triple_from_env(); - let py_version = if self.abi3 { None } else { Some(self.version) }; + let py_version = if self.implementation == PythonImplementation::CPython + && self.abi3 + && !self.is_free_threaded() + { + None + } else { + Some(self.version) + }; let abiflags = if self.is_free_threaded() { Some("t") } else { @@ -641,6 +663,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { writeln!(writer, "extra_build_script_line={}", line) @@ -889,6 +912,9 @@ pub struct CrossCompileConfig { /// The compile target triple (e.g. aarch64-unknown-linux-gnu) target: Triple, + + /// Python ABI flags, used to detect free-threaded Python builds. + abiflags: Option, } impl CrossCompileConfig { @@ -903,7 +929,7 @@ impl CrossCompileConfig { ) -> Result> { if env_vars.any() || Self::is_cross_compiling_from_to(host, target) { let lib_dir = env_vars.lib_dir_path()?; - let version = env_vars.parse_version()?; + let (version, abiflags) = env_vars.parse_version()?; let implementation = env_vars.parse_implementation()?; let target = target.clone(); @@ -912,6 +938,7 @@ impl CrossCompileConfig { version, implementation, target, + abiflags, })) } else { Ok(None) @@ -931,7 +958,9 @@ impl CrossCompileConfig { // Not cross-compiling to compile for 32-bit Python from windows 64-bit compatible |= target.operating_system == OperatingSystem::Windows - && host.operating_system == OperatingSystem::Windows; + && host.operating_system == OperatingSystem::Windows + && matches!(target.architecture, Architecture::X86_32(_)) + && host.architecture == Architecture::X86_64; // Not cross-compiling to compile for x86-64 Python from macOS arm64 and vice versa compatible |= target.operating_system == OperatingSystem::Darwin @@ -985,22 +1014,25 @@ impl CrossCompileEnvVars { } /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value - /// into `PythonVersion`. - fn parse_version(&self) -> Result> { - let version = self - .pyo3_cross_python_version - .as_ref() - .map(|os_string| { + /// into `PythonVersion` and ABI flags. + fn parse_version(&self) -> Result<(Option, Option)> { + match self.pyo3_cross_python_version.as_ref() { + Some(os_string) => { let utf8_str = os_string .to_str() .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid a UTF-8 string")?; - utf8_str + let (utf8_str, abiflags) = if let Some(version) = utf8_str.strip_suffix('t') { + (version, Some("t".to_string())) + } else { + (utf8_str, None) + }; + let version = utf8_str .parse() - .context("failed to parse PYO3_CROSS_PYTHON_VERSION") - }) - .transpose()?; - - Ok(version) + .context("failed to parse PYO3_CROSS_PYTHON_VERSION")?; + Ok((Some(version), abiflags)) + } + None => Ok((None, None)), + } } /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value @@ -1526,16 +1558,27 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result Result Option { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows(version, implementation, abi3, false, false, false).unwrap()) + Some( + default_lib_name_windows(version, implementation, abi3, false, false, gil_disabled) + .unwrap(), + ) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None, false).unwrap()) + Some(default_lib_name_unix(version, implementation, None, gil_disabled).unwrap()) } else { None } @@ -1902,7 +1951,14 @@ pub fn make_interpreter_config() -> Result { // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - let py_version = if interpreter_config.abi3 { + let gil_disabled = interpreter_config + .build_flags + .0 + .contains(&BuildFlag::Py_GIL_DISABLED); + let py_version = if interpreter_config.implementation == PythonImplementation::CPython + && interpreter_config.abi3 + && !gil_disabled + { None } else { Some(interpreter_config.version) @@ -1945,7 +2001,7 @@ fn unescape(escaped: &str) -> Vec { } } - bytes.push(unhex(chunk[0]) << 4 | unhex(chunk[1])); + bytes.push((unhex(chunk[0]) << 4) | unhex(chunk[1])); } bytes @@ -1971,6 +2027,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1999,6 +2056,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2020,6 +2078,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2046,6 +2105,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2068,6 +2128,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2170,6 +2231,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2199,6 +2261,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2225,6 +2288,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2248,6 +2312,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2271,6 +2336,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2305,6 +2371,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2339,6 +2406,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2373,6 +2441,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2409,6 +2478,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2702,7 +2772,7 @@ mod tests { assert_eq!( env_vars.parse_version().unwrap(), - Some(PythonVersion { major: 3, minor: 9 }) + (Some(PythonVersion { major: 3, minor: 9 }), None), ); let env_vars = CrossCompileEnvVars { @@ -2712,7 +2782,25 @@ mod tests { pyo3_cross_python_implementation: None, }; - assert_eq!(env_vars.parse_version().unwrap(), None); + assert_eq!(env_vars.parse_version().unwrap(), (None, None)); + + let env_vars = CrossCompileEnvVars { + pyo3_cross: None, + pyo3_cross_lib_dir: None, + pyo3_cross_python_version: Some("3.13t".into()), + pyo3_cross_python_implementation: None, + }; + + assert_eq!( + env_vars.parse_version().unwrap(), + ( + Some(PythonVersion { + major: 3, + minor: 13 + }), + Some("t".into()) + ), + ); let env_vars = CrossCompileEnvVars { pyo3_cross: None, @@ -2738,6 +2826,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2760,6 +2849,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2795,6 +2885,11 @@ mod tests { version: Some(interpreter_config.version), implementation: Some(interpreter_config.implementation), target: triple!("x86_64-unknown-linux-gnu"), + abiflags: if interpreter_config.is_free_threaded() { + Some("t".into()) + } else { + None + }, }; let sysconfigdata_path = match find_sysconfigdata(&cross) { @@ -2819,6 +2914,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2894,6 +2990,16 @@ mod tests { .is_none()); } + #[test] + fn test_is_cross_compiling_from_to() { + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("aarch64-pc-windows-msvc") + ) + .unwrap() + .is_some()); + } + #[test] fn test_run_python_script() { // as above, this should be okay in CI where Python is presumed installed @@ -2933,6 +3039,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), @@ -2972,6 +3079,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3019,6 +3127,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3052,6 +3161,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3070,6 +3180,7 @@ mod tests { version: None, implementation: None, target: triple!("x86_64-unknown-linux-gnu"), + abiflags: None, }) .unwrap_err(); diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 642fdf1659f..db7f3322844 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -74,6 +74,44 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap(); + _add_python_framework_link_args( + &interpreter_config, + &impl_::target_triple_from_env(), + impl_::is_linking_libpython(), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!( + writer, + "cargo:rustc-link-arg=-Wl,-rpath,{}", + framework_prefix + ) + .unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. @@ -156,6 +194,10 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 79 { println!("cargo:rustc-cfg=diagnostic_namespace"); } + + if rustc_minor_version >= 85 { + println!("cargo:rustc-cfg=fn_ptr_eq"); + } } /// Registers `pyo3`s config names as reachable cfg expressions @@ -180,6 +222,7 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(diagnostic_namespace)"); println!("cargo:rustc-check-cfg=cfg(c_str_lit)"); println!("cargo:rustc-check-cfg=cfg(rustc_has_once_lock)"); + println!("cargo:rustc-check-cfg=cfg(fn_ptr_eq)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) @@ -296,4 +339,49 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } } diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index ddbb489e2b9..fe8b3b25513 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.23.3" +version = "0.23.4" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -42,7 +42,7 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.3", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.4", features = ["resolve-config"] } [lints] workspace = true diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 5ae73122501..d220949a875 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -41,13 +41,13 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3-ffi] -version = "0.23.3" +version = "0.23.4" features = ["extension-module"] [build-dependencies] # This is only necessary if you need to configure your build based on # the Python version or the compile-time configuration for the interpreter. -pyo3_build_config = "0.23.3" +pyo3_build_config = "0.23.4" ``` If you need to use conditional compilation based on Python version or how diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 1899545011a..ce6c9b94735 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -17,7 +17,6 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_ extern "C" { #[cfg(all( not(PyPy), - not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index 83295e58f61..477ad02b747 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,5 +1,7 @@ use crate::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +#[cfg(not(all(Py_3_11, GraalPy)))] +use std::os::raw::c_char; +use std::os::raw::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; diff --git a/pyo3-ffi/src/cpython/complexobject.rs b/pyo3-ffi/src/cpython/complexobject.rs index 255f9c27034..4cc86db5667 100644 --- a/pyo3-ffi/src/cpython/complexobject.rs +++ b/pyo3-ffi/src/cpython/complexobject.rs @@ -19,7 +19,6 @@ pub struct Py_complex { #[repr(C)] pub struct PyComplexObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub cval: Py_complex, } diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs index 8c7ee88543d..e7caa441c5d 100644 --- a/pyo3-ffi/src/cpython/floatobject.rs +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -6,7 +6,6 @@ use std::os::raw::c_double; #[repr(C)] pub struct PyFloatObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub ob_fval: c_double, } diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index 17348b2f7bd..4be310a8c88 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -2,7 +2,7 @@ use crate::object::*; use crate::PyFrameObject; #[cfg(not(any(PyPy, GraalPy)))] use crate::_PyErr_StackItem; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(GraalPy)))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index 963ddfbea87..694e6bc4290 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -2,7 +2,7 @@ use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] #[repr(C)] pub struct PyListObject { pub ob_base: PyVarObject, @@ -10,7 +10,7 @@ pub struct PyListObject { pub allocated: Py_ssize_t, } -#[cfg(any(PyPy, GraalPy))] +#[cfg(PyPy)] pub struct PyListObject { pub ob_base: PyObject, } diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 35ddf25a2de..75eef11aae3 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -211,8 +211,6 @@ pub type printfunc = #[derive(Debug)] pub struct PyTypeObject { pub ob_base: object::PyVarObject, - #[cfg(GraalPy)] - pub ob_size: Py_ssize_t, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, pub tp_itemsize: Py_ssize_t, diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 3e0270ddc8f..98a19abeb81 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -1,3 +1,4 @@ +#[cfg(not(all(Py_3_11, GraalPy)))] use libc::size_t; use std::os::raw::c_int; diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index ca08b44a95c..c6e10e5f07b 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -6,19 +6,19 @@ use crate::Py_ssize_t; #[derive(Debug)] pub struct PyBaseExceptionObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, } @@ -134,19 +134,19 @@ pub struct PyOSErrorObject { #[derive(Debug)] pub struct PyStopIterationObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, pub value: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 1d988d2bef0..9616d4372cc 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -5,7 +5,6 @@ use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, - #[cfg(not(GraalPy))] pub ob_item: [*mut PyObject; 1], } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 1414b4ceb38..fae626b8d25 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,4 +1,4 @@ -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] use crate::Py_hash_t; use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t}; use libc::wchar_t; @@ -250,9 +250,8 @@ impl From for u32 { #[repr(C)] pub struct PyASCIIObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub length: Py_ssize_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hash: Py_hash_t, /// A bit field with various properties. /// @@ -265,9 +264,8 @@ pub struct PyASCIIObject { /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; - #[cfg(not(GraalPy))] pub state: u32, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } @@ -379,11 +377,9 @@ impl PyASCIIObject { #[repr(C)] pub struct PyCompactUnicodeObject { pub _base: PyASCIIObject, - #[cfg(not(GraalPy))] pub utf8_length: Py_ssize_t, - #[cfg(not(GraalPy))] pub utf8: *mut c_char, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr_length: Py_ssize_t, } @@ -398,7 +394,6 @@ pub union PyUnicodeObjectData { #[repr(C)] pub struct PyUnicodeObject { pub _base: PyCompactUnicodeObject, - #[cfg(not(GraalPy))] pub data: PyUnicodeObjectData, } diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 76d12151afc..7f2d7958364 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -9,13 +9,12 @@ use crate::PyCapsule_Import; #[cfg(GraalPy)] use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -#[cfg(not(GraalPy))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr; use std::sync::Once; use std::{cell::UnsafeCell, ffi::CStr}; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; @@ -27,13 +26,10 @@ const _PyDateTime_DATETIME_DATASIZE: usize = 10; /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub days: c_int, - #[cfg(not(GraalPy))] pub seconds: c_int, - #[cfg(not(GraalPy))] pub microseconds: c_int, } @@ -56,19 +52,17 @@ pub struct _PyDateTime_BaseTime { /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -77,11 +71,11 @@ pub struct PyDateTime_Time { /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } @@ -101,19 +95,17 @@ pub struct _PyDateTime_BaseDateTime { /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseDateTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -127,7 +119,7 @@ pub struct PyDateTime_DateTime { pub unsafe fn PyDateTime_GET_YEAR(o: *mut PyObject) -> c_int { // This should work for Date or DateTime let data = (*(o as *mut PyDateTime_Date)).data; - c_int::from(data[0]) << 8 | c_int::from(data[1]) + (c_int::from(data[0]) << 8) | c_int::from(data[1]) } #[inline] diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index fc3484be102..3f086ac1e92 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -129,6 +129,9 @@ pub struct PyVarObject { pub ob_base: PyObject, #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, + // On GraalPy the field is physically there, but not always populated. We hide it to prevent accidental misuse + #[cfg(GraalPy)] + pub _ob_size_graalpy: Py_ssize_t, } // skipped private _PyVarObject_CAST diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index f42f9730f1b..4f14e04a695 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,9 @@ -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::{c_char, c_void}; +use std::os::raw::c_char; +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +use std::os::raw::c_void; use std::os::raw::{c_int, c_ulong}; @@ -10,7 +12,7 @@ extern "C" { // skipped non-limited _Py_HashPointer // skipped non-limited _Py_HashPointerRaw - #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] pub fn _Py_HashBytes(src: *const c_void, len: Py_ssize_t) -> Py_hash_t; } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 38ec968d71d..704c263cb47 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.23.3" +version = "0.23.4" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -17,7 +17,7 @@ rust-version = "1.63" [dependencies] heck = "0.5" proc-macro2 = { version = "1.0.60", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.3", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.4", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] @@ -26,7 +26,7 @@ default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.3" } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.4" } [lints] workspace = true diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 14c8755e9be..565c54da1f3 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -3,6 +3,7 @@ use crate::utils::Ctx; use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::{ + ext::IdentExt, parenthesized, parse::{Parse, ParseStream}, parse_quote, @@ -323,11 +324,11 @@ impl<'a> Container<'a> { fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; - let struct_name = &self.name(); - let mut fields: Punctuated = Punctuated::new(); + let struct_name = self.name(); + let mut fields: Punctuated = Punctuated::new(); for field in struct_fields { - let ident = &field.ident; - let field_name = ident.to_string(); + let ident = field.ident; + let field_name = ident.unraw().to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 93596611f18..7697ae4451f 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -19,8 +19,9 @@ use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyfunction::ConstructorAttribute; use crate::pyimpl::{gen_py_const, get_cfg_attributes, PyClassMethodsType}; use crate::pymethod::{ - impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, - SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, __RICHCMP__, __STR__, + impl_py_class_attribute, impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, + MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, + __RICHCMP__, __STR__, }; use crate::pyversions::is_abi3_before; use crate::utils::{self, apply_renaming_rule, Ctx, LitCStr, PythonDoc}; @@ -1185,34 +1186,30 @@ fn impl_complex_enum_variant_cls( } fn impl_complex_enum_variant_match_args( - ctx: &Ctx, + ctx @ Ctx { pyo3_path, .. }: &Ctx, variant_cls_type: &syn::Type, field_names: &mut Vec, -) -> (MethodAndMethodDef, syn::ImplItemConst) { +) -> syn::Result<(MethodAndMethodDef, syn::ImplItemFn)> { let ident = format_ident!("__match_args__"); - let match_args_const_impl: syn::ImplItemConst = { - let args_tp = field_names.iter().map(|_| { - quote! { &'static str } - }); + let mut match_args_impl: syn::ImplItemFn = { parse_quote! { - #[allow(non_upper_case_globals)] - const #ident: ( #(#args_tp,)* ) = ( - #(stringify!(#field_names),)* - ); + #[classattr] + fn #ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'_, #pyo3_path::types::PyTuple>> { + #pyo3_path::types::PyTuple::new::<&str, _>(py, [ + #(stringify!(#field_names),)* + ]) + } } }; - let spec = ConstSpec { - rust_ident: ident, - attributes: ConstAttributes { - is_class_attr: true, - name: None, - }, - }; - - let variant_match_args = gen_py_const(variant_cls_type, &spec, ctx); + let spec = FnSpec::parse( + &mut match_args_impl.sig, + &mut match_args_impl.attrs, + Default::default(), + )?; + let variant_match_args = impl_py_class_attribute(variant_cls_type, &spec, ctx)?; - (variant_match_args, match_args_const_impl) + Ok((variant_match_args, match_args_impl)) } fn impl_complex_enum_struct_variant_cls( @@ -1260,7 +1257,7 @@ fn impl_complex_enum_struct_variant_cls( } let (variant_match_args, match_args_const_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; field_getters.push(variant_match_args); @@ -1268,6 +1265,7 @@ fn impl_complex_enum_struct_variant_cls( #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { + #[allow(clippy::too_many_arguments)] fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#fields_with_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident { #(#field_names,)* }; <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) @@ -1434,7 +1432,7 @@ fn impl_complex_enum_tuple_variant_cls( slots.push(variant_getitem); let (variant_match_args, match_args_method_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; field_getters.push(variant_match_args); @@ -1442,6 +1440,7 @@ fn impl_complex_enum_tuple_variant_cls( #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { + #[allow(clippy::too_many_arguments)] fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#field_names : #field_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident ( #(#field_names,)* ); <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) @@ -2309,10 +2308,9 @@ impl<'a> PyClassImplsBuilder<'a> { let assertions = if attr.options.unsendable.is_some() { TokenStream::new() } else { - let assert = quote_spanned! { cls.span() => assert_pyclass_sync::<#cls>(); }; + let assert = quote_spanned! { cls.span() => #pyo3_path::impl_::pyclass::assert_pyclass_sync::<#cls>(); }; quote! { const _: () = { - use #pyo3_path::impl_::pyclass::*; #assert }; } @@ -2352,7 +2350,7 @@ impl<'a> PyClassImplsBuilder<'a> { static DOC: #pyo3_path::sync::GILOnceCell<::std::borrow::Cow<'static, ::std::ffi::CStr>> = #pyo3_path::sync::GILOnceCell::new(); DOC.get_or_try_init(py, || { let collector = PyClassImplCollector::::new(); - build_pyclass_doc(<#cls as #pyo3_path::PyTypeInfo>::NAME, #doc, collector.new_text_signature()) + build_pyclass_doc(::NAME, #doc, collector.new_text_signature()) }).map(::std::ops::Deref::deref) } diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 560c3c9dcc1..58574c8bcc8 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -527,7 +527,7 @@ fn impl_clear_slot(cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx) -> syn::Result }) } -fn impl_py_class_attribute( +pub(crate) fn impl_py_class_attribute( cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx, diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index 47b82605bd1..d961b4c0acd 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -17,6 +17,7 @@ pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); quote_spanned! { *output_span => { let obj = #obj; + #[allow(clippy::useless_conversion)] #pyo3_path::impl_::wrap::converter(&obj).wrap(obj).map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) }} } diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index a44758b37f5..55961376c79 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.23.3" +version = "0.23.4" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -22,7 +22,7 @@ experimental-async = ["pyo3-macros-backend/experimental-async"] proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.23.3" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.23.4" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 771e6f8a045..f74a09ed943 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.23.3" +version = "0.23.4" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 90f9c69761d..04febb43b78 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -48,20 +48,23 @@ use crate::sync::GILOnceCell; use crate::types::any::PyAnyMethods; #[cfg(not(Py_LIMITED_API))] use crate::types::datetime::timezone_from_offset; +#[cfg(Py_LIMITED_API)] +use crate::types::IntoPyDict; use crate::types::PyNone; #[cfg(not(Py_LIMITED_API))] use crate::types::{ timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, PyTzInfo, PyTzInfoAccess, }; -use crate::{ffi, Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python}; +use crate::{ffi, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyObject, PyResult, Python}; #[cfg(Py_LIMITED_API)] use crate::{intern, DowncastError}; #[allow(deprecated)] use crate::{IntoPy, ToPyObject}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ - DateTime, Datelike, Duration, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, + DateTime, Datelike, Duration, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, Offset, + TimeZone, Timelike, }; #[allow(deprecated)] @@ -418,11 +421,14 @@ impl ToPyObject for DateTime { #[allow(deprecated)] impl IntoPy for DateTime { fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() + self.to_object(py) } } -impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime { +impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime +where + Tz: IntoPyObject<'py>, +{ #[cfg(Py_LIMITED_API)] type Target = PyAny; #[cfg(not(Py_LIMITED_API))] @@ -436,7 +442,10 @@ impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime { } } -impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime { +impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime +where + Tz: IntoPyObject<'py>, +{ #[cfg(Py_LIMITED_API)] type Target = PyAny; #[cfg(not(Py_LIMITED_API))] @@ -445,7 +454,11 @@ impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime { type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - let tz = self.offset().fix().into_pyobject(py)?; + let tz = self.timezone().into_bound_py_any(py)?; + + #[cfg(not(Py_LIMITED_API))] + let tz = tz.downcast()?; + let DateArgs { year, month, day } = (&self.naive_local().date()).into(); let TimeArgs { hour, @@ -455,14 +468,21 @@ impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime { truncated_leap_second, } = (&self.naive_local().time()).into(); + let fold = matches!( + self.timezone().offset_from_local_datetime(&self.naive_local()), + LocalResult::Ambiguous(_, latest) if self.offset().fix() == latest.fix() + ); + #[cfg(not(Py_LIMITED_API))] - let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, Some(&tz))?; + let datetime = + PyDateTime::new_with_fold(py, year, month, day, hour, min, sec, micro, Some(tz), fold)?; #[cfg(Py_LIMITED_API)] let datetime = DatetimeTypes::try_get(py).and_then(|dt| { - dt.datetime - .bind(py) - .call1((year, month, day, hour, min, sec, micro, tz)) + dt.datetime.bind(py).call( + (year, month, day, hour, min, sec, micro, tz), + Some(&[("fold", fold as u8)].into_py_dict(py)?), + ) })?; if truncated_leap_second { @@ -493,12 +513,26 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime Ok(value), + LocalResult::Ambiguous(earliest, latest) => { + #[cfg(not(Py_LIMITED_API))] + let fold = dt.get_fold(); + + #[cfg(Py_LIMITED_API)] + let fold = dt.getattr(intern!(dt.py(), "fold"))?.extract::()? > 0; + + if fold { + Ok(latest) + } else { + Ok(earliest) + } + } + LocalResult::None => Err(PyValueError::new_err(format!( + "The datetime {:?} contains an incompatible timezone", dt - )) - }) + ))), + } } } @@ -971,8 +1005,12 @@ mod tests { // Also check that trying to convert an out of bound value errors. Python::with_gil(|py| { - assert!(Duration::min_value().into_pyobject(py).is_err()); - assert!(Duration::max_value().into_pyobject(py).is_err()); + // min_value and max_value were deprecated in chrono 0.4.39 + #[allow(deprecated)] + { + assert!(Duration::min_value().into_pyobject(py).is_err()); + assert!(Duration::max_value().into_pyobject(py).is_err()); + } }); } @@ -1170,6 +1208,35 @@ mod tests { }) } + #[test] + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn test_pyo3_datetime_into_pyobject_tz() { + Python::with_gil(|py| { + let datetime = NaiveDate::from_ymd_opt(2024, 12, 11) + .unwrap() + .and_hms_opt(23, 3, 13) + .unwrap() + .and_local_timezone(chrono_tz::Tz::Europe__London) + .unwrap(); + let datetime = datetime.into_pyobject(py).unwrap(); + let py_datetime = new_py_datetime_ob( + py, + "datetime", + ( + 2024, + 12, + 11, + 23, + 3, + 13, + 0, + python_zoneinfo(py, "Europe/London"), + ), + ); + assert_eq!(datetime.compare(&py_datetime).unwrap(), Ordering::Equal); + }) + } + #[test] fn test_pyo3_datetime_frompyobject_utc() { Python::with_gil(|py| { @@ -1373,6 +1440,16 @@ mod tests { .unwrap() } + #[cfg(all(Py_3_9, feature = "chrono-tz", not(windows)))] + fn python_zoneinfo<'py>(py: Python<'py>, timezone: &str) -> Bound<'py, PyAny> { + py.import("zoneinfo") + .unwrap() + .getattr("ZoneInfo") + .unwrap() + .call1((timezone,)) + .unwrap() + } + #[cfg(not(any(target_arch = "wasm32", Py_GIL_DISABLED)))] mod proptests { use super::*; diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index bb1a74c1519..60a3bab4918 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -98,6 +98,10 @@ impl FromPyObject<'_> for Tz { #[cfg(all(test, not(windows)))] // Troubles loading timezones on Windows mod tests { use super::*; + use crate::prelude::PyAnyMethods; + use crate::Python; + use chrono::{DateTime, Utc}; + use chrono_tz::Tz; #[test] fn test_frompyobject() { @@ -114,6 +118,54 @@ mod tests { }); } + #[test] + fn test_ambiguous_datetime_to_pyobject() { + let dates = [ + DateTime::::from_str("2020-10-24 23:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 00:00:00 UTC").unwrap(), + DateTime::::from_str("2020-10-25 01:00:00 UTC").unwrap(), + ]; + + let dates = dates.map(|dt| dt.with_timezone(&Tz::Europe__London)); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + + let dates = Python::with_gil(|py| { + let pydates = dates.map(|dt| dt.into_pyobject(py).unwrap()); + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("hour").unwrap().extract::().unwrap()), + [0, 1, 1] + ); + + assert_eq!( + pydates + .clone() + .map(|dt| dt.getattr("fold").unwrap().extract::().unwrap() > 0), + [false, false, true] + ); + + pydates.map(|dt| dt.extract::>().unwrap()) + }); + + assert_eq!( + dates.map(|dt| dt.to_string()), + [ + "2020-10-25 00:00:00 BST", + "2020-10-25 01:00:00 BST", + "2020-10-25 01:00:00 GMT" + ] + ); + } + #[test] #[cfg(not(Py_GIL_DISABLED))] // https://github.com/python/cpython/issues/116738#issuecomment-2404360445 fn test_into_pyobject() { diff --git a/src/err/err_state.rs b/src/err/err_state.rs index 3b9e9800b6e..98be633e91c 100644 --- a/src/err/err_state.rs +++ b/src/err/err_state.rs @@ -43,7 +43,13 @@ impl PyErrState { } pub(crate) fn normalized(normalized: PyErrStateNormalized) -> Self { - Self::from_inner(PyErrStateInner::Normalized(normalized)) + let state = Self::from_inner(PyErrStateInner::Normalized(normalized)); + // This state is already normalized, by completing the Once immediately we avoid + // reaching the `py.allow_threads` in `make_normalized` which is less efficient + // and introduces a GIL switch which could deadlock. + // See https://github.com/PyO3/pyo3/issues/4764 + state.normalized.call_once(|| {}); + state } pub(crate) fn restore(self, py: Python<'_>) { diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 58d0c93c240..e6cbaec86f5 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -21,6 +21,7 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr::null_mut; use super::trampoline; +use crate::internal_tricks::{clear_eq, traverse_eq}; /// Python 3.8 and up - __ipow__ has modulo argument correctly populated. #[cfg(Py_3_8)] @@ -364,7 +365,7 @@ unsafe fn call_super_traverse( // First find the current type by the current_traverse function loop { traverse = get_slot(ty, TP_TRAVERSE); - if traverse == Some(current_traverse) { + if traverse_eq(traverse, current_traverse) { break; } ty = get_slot(ty, TP_BASE); @@ -375,7 +376,7 @@ unsafe fn call_super_traverse( } // Get first base which has a different traverse function - while traverse == Some(current_traverse) { + while traverse_eq(traverse, current_traverse) { ty = get_slot(ty, TP_BASE); if ty.is_null() { break; @@ -429,7 +430,7 @@ unsafe fn call_super_clear( // First find the current type by the current_clear function loop { clear = ty.get_slot(TP_CLEAR); - if clear == Some(current_clear) { + if clear_eq(clear, current_clear) { break; } let base = ty.get_slot(TP_BASE); @@ -441,7 +442,7 @@ unsafe fn call_super_clear( } // Get first base which has a different clear function - while clear == Some(current_clear) { + while clear_eq(clear, current_clear) { let base = ty.get_slot(TP_BASE); if base.is_null() { break; diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 97b13aff2a8..dad3f7c4c03 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -1,4 +1,4 @@ -use crate::ffi::{Py_ssize_t, PY_SSIZE_T_MAX}; +use crate::ffi::{self, Py_ssize_t, PY_SSIZE_T_MAX}; pub struct PrivateMarker; macro_rules! private_decl { @@ -47,3 +47,31 @@ pub(crate) const fn ptr_from_ref(t: &T) -> *const T { pub(crate) fn ptr_from_mut(t: &mut T) -> *mut T { t as *mut T } + +// TODO: use ptr::fn_addr_eq on MSRV 1.85 +pub(crate) fn clear_eq(f: Option, g: ffi::inquiry) -> bool { + #[cfg(fn_ptr_eq)] + { + let Some(f) = f else { return false }; + std::ptr::fn_addr_eq(f, g) + } + + #[cfg(not(fn_ptr_eq))] + { + f == Some(g) + } +} + +// TODO: use ptr::fn_addr_eq on MSRV 1.85 +pub(crate) fn traverse_eq(f: Option, g: ffi::traverseproc) -> bool { + #[cfg(fn_ptr_eq)] + { + let Some(f) = f else { return false }; + std::ptr::fn_addr_eq(f, g) + } + + #[cfg(not(fn_ptr_eq))] + { + f == Some(g) + } +} diff --git a/src/tests/hygiene/pyclass.rs b/src/tests/hygiene/pyclass.rs index 4270da34be3..17c3ce41e11 100644 --- a/src/tests/hygiene/pyclass.rs +++ b/src/tests/hygiene/pyclass.rs @@ -92,6 +92,44 @@ pub enum TupleEnumEqOrd { Variant2(u32), } +#[crate::pyclass(crate = "crate")] +pub enum ComplexEnumManyVariantFields { + ManyStructFields { + field_1: u16, + field_2: u32, + field_3: u32, + field_4: i32, + field_5: u32, + field_6: u32, + field_7: u8, + field_8: u32, + field_9: i32, + field_10: u32, + field_11: u32, + field_12: u32, + field_13: u32, + field_14: i16, + field_15: u32, + }, + ManyTupleFields( + u16, + u32, + u32, + i32, + u32, + u32, + u8, + u32, + i32, + u32, + u32, + u32, + u32, + i16, + u32, + ), +} + #[crate::pyclass(str = "{x}, {y}, {z}")] #[pyo3(crate = "crate")] pub struct PointFmt { diff --git a/src/types/any.rs b/src/types/any.rs index d060c187631..8d1296ef15a 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -948,7 +948,7 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { } } - inner(self.py(), self.getattr(attr_name).map_err(Into::into)) + inner(self.py(), self.getattr(attr_name)) } fn getattr(&self, attr_name: N) -> PyResult> diff --git a/src/types/dict.rs b/src/types/dict.rs index b3c8e37962b..0d2e6ff335f 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -181,7 +181,8 @@ pub trait PyDictMethods<'py>: crate::sealed::Sealed { /// Iterates over the contents of this dictionary while holding a critical section on the dict. /// This is useful when the GIL is disabled and the dictionary is shared between threads. /// It is not guaranteed that the dictionary will not be modified during iteration when the - /// closure calls arbitrary Python code that releases the current critical section. + /// closure calls arbitrary Python code that releases the critical section held by the + /// iterator. Otherwise, the dictionary will not be modified during iteration. /// /// This method is a small performance optimization over `.iter().try_for_each()` when the /// nightly feature is not enabled because we cannot implement an optimised version of @@ -396,19 +397,26 @@ impl<'a, 'py> Borrowed<'a, 'py, PyDict> { /// Iterates over the contents of this dictionary without incrementing reference counts. /// /// # Safety - /// It must be known that this dictionary will not be modified during iteration. + /// It must be known that this dictionary will not be modified during iteration, + /// for example, when parsing arguments in a keyword arguments dictionary. pub(crate) unsafe fn iter_borrowed(self) -> BorrowedDictIter<'a, 'py> { BorrowedDictIter::new(self) } } fn dict_len(dict: &Bound<'_, PyDict>) -> Py_ssize_t { - #[cfg(any(not(Py_3_8), PyPy, GraalPy, Py_LIMITED_API))] + #[cfg(any(not(Py_3_8), PyPy, GraalPy, Py_LIMITED_API, Py_GIL_DISABLED))] unsafe { ffi::PyDict_Size(dict.as_ptr()) } - #[cfg(all(Py_3_8, not(PyPy), not(GraalPy), not(Py_LIMITED_API)))] + #[cfg(all( + Py_3_8, + not(PyPy), + not(GraalPy), + not(Py_LIMITED_API), + not(Py_GIL_DISABLED) + ))] unsafe { (*dict.as_ptr().cast::()).ma_used } @@ -429,8 +437,11 @@ enum DictIterImpl { } impl DictIterImpl { + #[deny(unsafe_op_in_unsafe_fn)] #[inline] - fn next<'py>( + /// Safety: the dict should be locked with a critical section on the free-threaded build + /// and otherwise not shared between threads in code that releases the GIL. + unsafe fn next_unchecked<'py>( &mut self, dict: &Bound<'py, PyDict>, ) -> Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -440,7 +451,7 @@ impl DictIterImpl { remaining, ppos, .. - } => crate::sync::with_critical_section(dict, || { + } => { let ma_used = dict_len(dict); // These checks are similar to what CPython does. @@ -470,20 +481,20 @@ impl DictIterImpl { let mut key: *mut ffi::PyObject = std::ptr::null_mut(); let mut value: *mut ffi::PyObject = std::ptr::null_mut(); - if unsafe { ffi::PyDict_Next(dict.as_ptr(), ppos, &mut key, &mut value) } != 0 { + if unsafe { ffi::PyDict_Next(dict.as_ptr(), ppos, &mut key, &mut value) != 0 } { *remaining -= 1; let py = dict.py(); // Safety: // - PyDict_Next returns borrowed values // - we have already checked that `PyDict_Next` succeeded, so we can assume these to be non-null Some(( - unsafe { key.assume_borrowed_unchecked(py) }.to_owned(), - unsafe { value.assume_borrowed_unchecked(py) }.to_owned(), + unsafe { key.assume_borrowed_unchecked(py).to_owned() }, + unsafe { value.assume_borrowed_unchecked(py).to_owned() }, )) } else { None } - }), + } } } @@ -504,7 +515,17 @@ impl<'py> Iterator for BoundDictIterator<'py> { #[inline] fn next(&mut self) -> Option { - self.inner.next(&self.dict) + #[cfg(Py_GIL_DISABLED)] + { + self.inner + .with_critical_section(&self.dict, |inner| unsafe { + inner.next_unchecked(&self.dict) + }) + } + #[cfg(not(Py_GIL_DISABLED))] + { + unsafe { self.inner.next_unchecked(&self.dict) } + } } #[inline] @@ -522,7 +543,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut accum = init; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { accum = f(accum, x); } accum @@ -539,7 +560,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut accum = init; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { accum = f(accum, x)? } R::from_output(accum) @@ -554,7 +575,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if !f(x) { return false; } @@ -571,7 +592,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if f(x) { return true; } @@ -588,7 +609,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { P: FnMut(&Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if predicate(&x) { return Some(x); } @@ -605,7 +626,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> Option, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if let found @ Some(_) = f(x) { return found; } @@ -623,7 +644,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut acc = 0; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if predicate(x) { return Some(acc); } diff --git a/src/types/list.rs b/src/types/list.rs index af2b557cba9..76da36d00b9 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -18,7 +18,7 @@ use crate::types::sequence::PySequenceMethods; /// [`Py`][crate::Py] or [`Bound<'py, PyList>`][Bound]. /// /// For APIs available on `list` objects, see the [`PyListMethods`] trait which is implemented for -/// [`Bound<'py, PyDict>`][Bound]. +/// [`Bound<'py, PyList>`][Bound]. #[repr(transparent)] pub struct PyList(PyAny); @@ -179,7 +179,9 @@ pub trait PyListMethods<'py>: crate::sealed::Sealed { /// # Safety /// /// Caller must verify that the index is within the bounds of the list. - #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] + /// On the free-threaded build, caller must verify they have exclusive access to the list + /// via a lock or by holding the innermost critical section on the list. + #[cfg(not(Py_LIMITED_API))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny>; /// Takes the slice `self[low:high]` and returns it as a new list. @@ -239,6 +241,17 @@ pub trait PyListMethods<'py>: crate::sealed::Sealed { /// Returns an iterator over this list's items. fn iter(&self) -> BoundListIterator<'py>; + /// Iterates over the contents of this list while holding a critical section on the list. + /// This is useful when the GIL is disabled and the list is shared between threads. + /// It is not guaranteed that the list will not be modified during iteration when the + /// closure calls arbitrary Python code that releases the critical section held by the + /// iterator. Otherwise, the list will not be modified during iteration. + /// + /// This is equivalent to for_each if the GIL is enabled. + fn locked_for_each(&self, closure: F) -> PyResult<()> + where + F: Fn(Bound<'py, PyAny>) -> PyResult<()>; + /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. fn sort(&self) -> PyResult<()>; @@ -302,7 +315,7 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { /// # Safety /// /// Caller must verify that the index is within the bounds of the list. - #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] + #[cfg(not(Py_LIMITED_API))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny> { // PyList_GET_ITEM return borrowed ptr; must make owned for safety (see #890). ffi::PyList_GET_ITEM(self.as_ptr(), index as Py_ssize_t) @@ -440,6 +453,14 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { BoundListIterator::new(self.clone()) } + /// Iterates over a list while holding a critical section, calling a closure on each item + fn locked_for_each(&self, closure: F) -> PyResult<()> + where + F: Fn(Bound<'py, PyAny>) -> PyResult<()>, + { + crate::sync::with_critical_section(self, || self.iter().try_for_each(closure)) + } + /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. fn sort(&self) -> PyResult<()> { err::error_on_minusone(self.py(), unsafe { ffi::PyList_Sort(self.as_ptr()) }) @@ -462,73 +483,332 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { } } +// New types for type checking when using BoundListIterator associated methods, like +// BoundListIterator::next_unchecked. +struct Index(usize); +struct Length(usize); + /// Used by `PyList::iter()`. pub struct BoundListIterator<'py> { list: Bound<'py, PyList>, - index: usize, - length: usize, + index: Index, + length: Length, } impl<'py> BoundListIterator<'py> { fn new(list: Bound<'py, PyList>) -> Self { - let length: usize = list.len(); - BoundListIterator { + Self { + index: Index(0), + length: Length(list.len()), list, - index: 0, - length, } } - unsafe fn get_item(&self, index: usize) -> Bound<'py, PyAny> { - #[cfg(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED))] - let item = self.list.get_item(index).expect("list.get failed"); - #[cfg(not(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED)))] - let item = self.list.get_item_unchecked(index); - item + /// # Safety + /// + /// On the free-threaded build, caller must verify they have exclusive + /// access to the list by holding a lock or by holding the innermost + /// critical section on the list. + #[inline] + #[cfg(not(Py_LIMITED_API))] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn next_unchecked( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let length = length.0.min(list.len()); + let my_index = index.0; + + if index.0 < length { + let item = unsafe { list.get_item_unchecked(my_index) }; + index.0 += 1; + Some(item) + } else { + None + } } -} -impl<'py> Iterator for BoundListIterator<'py> { - type Item = Bound<'py, PyAny>; + #[cfg(Py_LIMITED_API)] + fn next( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let length = length.0.min(list.len()); + let my_index = index.0; + if index.0 < length { + let item = list.get_item(my_index).expect("get-item failed"); + index.0 += 1; + Some(item) + } else { + None + } + } + + /// # Safety + /// + /// On the free-threaded build, caller must verify they have exclusive + /// access to the list by holding a lock or by holding the innermost + /// critical section on the list. #[inline] - fn next(&mut self) -> Option { - let length = self.length.min(self.list.len()); + #[cfg(not(Py_LIMITED_API))] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn next_back_unchecked( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let current_length = length.0.min(list.len()); + + if index.0 < current_length { + let item = unsafe { list.get_item_unchecked(current_length - 1) }; + length.0 = current_length - 1; + Some(item) + } else { + None + } + } - if self.index < length { - let item = unsafe { self.get_item(self.index) }; - self.index += 1; + #[inline] + #[cfg(Py_LIMITED_API)] + fn next_back( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let current_length = (length.0).min(list.len()); + + if index.0 < current_length { + let item = list.get_item(current_length - 1).expect("get-item failed"); + length.0 = current_length - 1; Some(item) } else { None } } + #[cfg(not(Py_LIMITED_API))] + fn with_critical_section( + &mut self, + f: impl FnOnce(&mut Index, &mut Length, &Bound<'py, PyList>) -> R, + ) -> R { + let Self { + index, + length, + list, + } = self; + crate::sync::with_critical_section(list, || f(index, length, list)) + } +} + +impl<'py> Iterator for BoundListIterator<'py> { + type Item = Bound<'py, PyAny>; + + #[inline] + fn next(&mut self) -> Option { + #[cfg(not(Py_LIMITED_API))] + { + self.with_critical_section(|index, length, list| unsafe { + Self::next_unchecked(index, length, list) + }) + } + #[cfg(Py_LIMITED_API)] + { + let Self { + index, + length, + list, + } = self; + Self::next(index, length, list) + } + } + #[inline] fn size_hint(&self) -> (usize, Option) { let len = self.len(); (len, Some(len)) } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn fold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + accum = f(accum, x); + } + accum + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, feature = "nightly"))] + fn try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: std::ops::Try, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + accum = f(accum, x)? + } + R::from_output(accum) + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn all(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if !f(x) { + return false; + } + } + true + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn any(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if f(x) { + return true; + } + } + false + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn find

(&mut self, mut predicate: P) -> Option + where + Self: Sized, + P: FnMut(&Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if predicate(&x) { + return Some(x); + } + } + None + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn find_map(&mut self, mut f: F) -> Option + where + Self: Sized, + F: FnMut(Self::Item) -> Option, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if let found @ Some(_) = f(x) { + return found; + } + } + None + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn position

(&mut self, mut predicate: P) -> Option + where + Self: Sized, + P: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + let mut acc = 0; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if predicate(x) { + return Some(acc); + } + acc += 1; + } + None + }) + } } impl DoubleEndedIterator for BoundListIterator<'_> { #[inline] fn next_back(&mut self) -> Option { - let length = self.length.min(self.list.len()); - - if self.index < length { - let item = unsafe { self.get_item(length - 1) }; - self.length = length - 1; - Some(item) - } else { - None + #[cfg(not(Py_LIMITED_API))] + { + self.with_critical_section(|index, length, list| unsafe { + Self::next_back_unchecked(index, length, list) + }) + } + #[cfg(Py_LIMITED_API)] + { + let Self { + index, + length, + list, + } = self; + Self::next_back(index, length, list) } } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn rfold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_back_unchecked(index, length, list) } { + accum = f(accum, x); + } + accum + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, feature = "nightly"))] + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: std::ops::Try, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_back_unchecked(index, length, list) } { + accum = f(accum, x)? + } + R::from_output(accum) + }) + } } impl ExactSizeIterator for BoundListIterator<'_> { fn len(&self) -> usize { - self.length.saturating_sub(self.index) + self.length.0.saturating_sub(self.index.0) } } @@ -558,7 +838,7 @@ mod tests { use crate::types::list::PyListMethods; use crate::types::sequence::PySequenceMethods; use crate::types::{PyList, PyTuple}; - use crate::{ffi, IntoPyObject, Python}; + use crate::{ffi, IntoPyObject, PyResult, Python}; #[test] fn test_new() { @@ -748,6 +1028,142 @@ mod tests { }); } + #[test] + fn test_iter_all() { + Python::with_gil(|py| { + let list = PyList::new(py, [true, true, true]).unwrap(); + assert!(list.iter().all(|x| x.extract::().unwrap())); + + let list = PyList::new(py, [true, false, true]).unwrap(); + assert!(!list.iter().all(|x| x.extract::().unwrap())); + }); + } + + #[test] + fn test_iter_any() { + Python::with_gil(|py| { + let list = PyList::new(py, [true, true, true]).unwrap(); + assert!(list.iter().any(|x| x.extract::().unwrap())); + + let list = PyList::new(py, [true, false, true]).unwrap(); + assert!(list.iter().any(|x| x.extract::().unwrap())); + + let list = PyList::new(py, [false, false, false]).unwrap(); + assert!(!list.iter().any(|x| x.extract::().unwrap())); + }); + } + + #[test] + fn test_iter_find() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, ["hello", "world"]).unwrap(); + assert_eq!( + Some("world".to_string()), + list.iter() + .find(|v| v.extract::().unwrap() == "world") + .map(|v| v.extract::().unwrap()) + ); + assert_eq!( + None, + list.iter() + .find(|v| v.extract::().unwrap() == "foobar") + .map(|v| v.extract::().unwrap()) + ); + }); + } + + #[test] + fn test_iter_position() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, ["hello", "world"]).unwrap(); + assert_eq!( + Some(1), + list.iter() + .position(|v| v.extract::().unwrap() == "world") + ); + assert_eq!( + None, + list.iter() + .position(|v| v.extract::().unwrap() == "foobar") + ); + }); + } + + #[test] + fn test_iter_fold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .fold(0, |acc, v| acc + v.extract::().unwrap()); + assert_eq!(sum, 6); + }); + } + + #[test] + fn test_iter_fold_out_of_bounds() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list.iter().fold(0, |_, _| { + // clear the list to create a pathological fold operation + // that mutates the list as it processes it + for _ in 0..3 { + list.del_item(0).unwrap(); + } + -5 + }); + assert_eq!(sum, -5); + assert!(list.len() == 0); + }); + } + + #[test] + fn test_iter_rfold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .rfold(0, |acc, v| acc + v.extract::().unwrap()); + assert_eq!(sum, 6); + }); + } + + #[test] + fn test_iter_try_fold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .try_fold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .unwrap(); + assert_eq!(sum, 6); + + let list = PyList::new(py, ["foo", "bar"]).unwrap(); + assert!(list + .iter() + .try_fold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .is_err()); + }); + } + + #[test] + fn test_iter_try_rfold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .try_rfold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .unwrap(); + assert_eq!(sum, 6); + + let list = PyList::new(py, ["foo", "bar"]).unwrap(); + assert!(list + .iter() + .try_rfold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .is_err()); + }); + } + #[test] fn test_into_iter() { Python::with_gil(|py| { @@ -877,7 +1293,7 @@ mod tests { }); } - #[cfg(not(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED)))] + #[cfg(not(Py_LIMITED_API))] #[test] fn test_list_get_item_unchecked_sanity() { Python::with_gil(|py| { diff --git a/src/types/mod.rs b/src/types/mod.rs index d84f099e773..39ab3fd501e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -23,7 +23,7 @@ pub use self::float::{PyFloat, PyFloatMethods}; pub use self::frame::PyFrame; pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder, PyFrozenSetMethods}; pub use self::function::PyCFunction; -#[cfg(all(not(Py_LIMITED_API), not(all(PyPy, not(Py_3_8))), not(GraalPy)))] +#[cfg(all(not(Py_LIMITED_API), not(all(PyPy, not(Py_3_8)))))] pub use self::function::PyFunction; pub use self::iterator::PyIterator; pub use self::list::{PyList, PyListMethods}; diff --git a/src/types/module.rs b/src/types/module.rs index fd7299cb084..99dc1be233b 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -443,7 +443,7 @@ impl<'py> PyModuleMethods<'py> for Bound<'py, PyModule> { Err(err) => { if err.is_instance_of::(self.py()) { let l = PyList::empty(self.py()); - self.setattr(__all__, &l).map_err(PyErr::from)?; + self.setattr(__all__, &l)?; Ok(l) } else { Err(err) diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index e4e80e90263..4aba1d4723e 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -68,4 +68,5 @@ fn test_compile_errors() { #[cfg(all(not(Py_LIMITED_API), Py_3_11))] t.compile_fail("tests/ui/invalid_base_class.rs"); t.pass("tests/ui/ambiguous_associated_items.rs"); + t.pass("tests/ui/pyclass_probe.rs"); } diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 344a47acf72..2192caf1f7c 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -648,3 +648,41 @@ fn test_transparent_from_py_with() { assert_eq!(result, expected); }); } + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +pub struct WithKeywordAttr { + r#box: usize, +} + +#[pyclass] +pub struct WithKeywordAttrC { + #[pyo3(get)] + r#box: usize, +} + +#[test] +fn test_with_keyword_attr() { + Python::with_gil(|py| { + let cls = WithKeywordAttrC { r#box: 3 }.into_pyobject(py).unwrap(); + let result = cls.extract::().unwrap(); + let expected = WithKeywordAttr { r#box: 3 }; + assert_eq!(result, expected); + }); +} + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +pub struct WithKeywordItem { + #[pyo3(item)] + r#box: usize, +} + +#[test] +fn test_with_keyword_item() { + Python::with_gil(|py| { + let dict = PyDict::new(py); + dict.set_item("box", 3).unwrap(); + let result = dict.extract::().unwrap(); + let expected = WithKeywordItem { r#box: 3 }; + assert_eq!(result, expected); + }); +} diff --git a/tests/ui/invalid_cancel_handle.stderr b/tests/ui/invalid_cancel_handle.stderr index bd2b588df32..90a8caa4229 100644 --- a/tests/ui/invalid_cancel_handle.stderr +++ b/tests/ui/invalid_cancel_handle.stderr @@ -42,7 +42,7 @@ error[E0277]: the trait bound `CancelHandle: PyFunctionArgument<'_, '_>` is not --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `PyClass` is not implemented for `CancelHandle`, which is required by `CancelHandle: PyFunctionArgument<'_, '_>` + | ^^^^ the trait `PyClass` is not implemented for `CancelHandle` | = help: the trait `PyClass` is implemented for `pyo3::coroutine::Coroutine` = note: required for `CancelHandle` to implement `FromPyObject<'_>` @@ -61,7 +61,7 @@ error[E0277]: the trait bound `CancelHandle: PyFunctionArgument<'_, '_>` is not --> tests/ui/invalid_cancel_handle.rs:20:50 | 20 | async fn missing_cancel_handle_attribute(_param: pyo3::coroutine::CancelHandle) {} - | ^^^^ the trait `Clone` is not implemented for `CancelHandle`, which is required by `CancelHandle: PyFunctionArgument<'_, '_>` + | ^^^^ the trait `Clone` is not implemented for `CancelHandle` | = help: the following other types implement trait `PyFunctionArgument<'a, 'py>`: &'a mut pyo3::coroutine::Coroutine diff --git a/tests/ui/invalid_property_args.stderr b/tests/ui/invalid_property_args.stderr index f2fea2a1dd5..5ef01fa210a 100644 --- a/tests/ui/invalid_property_args.stderr +++ b/tests/ui/invalid_property_args.stderr @@ -52,7 +52,7 @@ error[E0277]: `PhantomData` cannot be converted to a Python object 45 | value: ::std::marker::PhantomData, | ^ required by `#[pyo3(get)]` to create a readable property from a field of type `PhantomData` | - = help: the trait `IntoPyObject<'_>` is not implemented for `PhantomData`, which is required by `for<'py> PhantomData: PyO3GetField<'py>` + = help: the trait `IntoPyObject<'_>` is not implemented for `PhantomData` = note: implement `IntoPyObject` for `&PhantomData` or `IntoPyObject + Clone` for `PhantomData` to define the conversion = help: the following other types implement trait `IntoPyObject<'py>`: &&'a T diff --git a/tests/ui/invalid_pyclass_args.stderr b/tests/ui/invalid_pyclass_args.stderr index d1335e0f1a1..e4d415ce177 100644 --- a/tests/ui/invalid_pyclass_args.stderr +++ b/tests/ui/invalid_pyclass_args.stderr @@ -100,21 +100,21 @@ error: expected one of: `get`, `set`, `name` 85 | #[pyo3(pop)] | ^^^ -error: invalid format string: expected `'}'` but string was terminated +error: invalid format string: expected `}` but string was terminated --> tests/ui/invalid_pyclass_args.rs:105:19 | 105 | #[pyclass(str = "{")] - | -^ expected `'}'` in format string + | -^ expected `}` in format string | | | because of this opening brace | = note: if you intended to print `{`, you can escape it using `{{` -error: invalid format string: expected `'}'`, found `'$'` +error: invalid format string: expected `}`, found `$` --> tests/ui/invalid_pyclass_args.rs:109:19 | 109 | #[pyclass(str = "{$}")] - | -^ expected `'}'` in format string + | -^ expected `}` in format string | | | because of this opening brace | diff --git a/tests/ui/invalid_pyfunctions.stderr b/tests/ui/invalid_pyfunctions.stderr index 9b35b8dd9fe..4e537ae1ea4 100644 --- a/tests/ui/invalid_pyfunctions.stderr +++ b/tests/ui/invalid_pyfunctions.stderr @@ -51,7 +51,7 @@ error[E0277]: the trait bound `&str: From tests/ui/invalid_pyfunctions.rs:36:14 | 36 | _string: &str, - | ^ the trait `From>` is not implemented for `&str`, which is required by `BoundRef<'_, '_, pyo3::types::PyModule>: Into<_>` + | ^ the trait `From>` is not implemented for `&str` | = help: the following other types implement trait `From`: `String` implements `From<&String>` diff --git a/tests/ui/invalid_pymethod_receiver.stderr b/tests/ui/invalid_pymethod_receiver.stderr index 9c998403194..7d81c0fae91 100644 --- a/tests/ui/invalid_pymethod_receiver.stderr +++ b/tests/ui/invalid_pymethod_receiver.stderr @@ -2,7 +2,7 @@ error[E0277]: the trait bound `i32: TryFrom>` is not s --> tests/ui/invalid_pymethod_receiver.rs:8:44 | 8 | fn method_with_invalid_self_type(_slf: i32, _py: Python<'_>, _index: u32) {} - | ^^^ the trait `From>` is not implemented for `i32`, which is required by `i32: TryFrom>` + | ^^^ the trait `From>` is not implemented for `i32` | = help: the following other types implement trait `From`: `i32` implements `From` diff --git a/tests/ui/invalid_pymethods.stderr b/tests/ui/invalid_pymethods.stderr index 845b79ed59a..eb8f429c937 100644 --- a/tests/ui/invalid_pymethods.stderr +++ b/tests/ui/invalid_pymethods.stderr @@ -183,7 +183,7 @@ error[E0277]: the trait bound `i32: From>` is not satis --> tests/ui/invalid_pymethods.rs:46:45 | 46 | fn classmethod_wrong_first_argument(_x: i32) -> Self { - | ^^^ the trait `From>` is not implemented for `i32`, which is required by `BoundRef<'_, '_, PyType>: Into<_>` + | ^^^ the trait `From>` is not implemented for `i32` | = help: the following other types implement trait `From`: `i32` implements `From` diff --git a/tests/ui/invalid_result_conversion.stderr b/tests/ui/invalid_result_conversion.stderr index 18667138954..f782e00828f 100644 --- a/tests/ui/invalid_result_conversion.stderr +++ b/tests/ui/invalid_result_conversion.stderr @@ -2,7 +2,7 @@ error[E0277]: the trait bound `PyErr: From` is not satisfied --> tests/ui/invalid_result_conversion.rs:22:25 | 22 | fn should_not_work() -> Result<(), MyError> { - | ^^^^^^ the trait `From` is not implemented for `PyErr`, which is required by `MyError: Into` + | ^^^^^^ the trait `From` is not implemented for `PyErr` | = help: the following other types implement trait `From`: `PyErr` implements `From` diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index 8007c2f4e54..5d05b3de059 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -6,7 +6,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | | | required by a bound introduced by this call | - = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>`, which is required by `{closure@$DIR/tests/ui/not_send.rs:4:22: 4:24}: Ungil` + = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index 52a4e6a7208..eec2b644e9a 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -9,7 +9,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe 10 | | }); | |_________^ `*mut pyo3::Python<'static>` cannot be shared between threads safely | - = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>`, which is required by `{closure@$DIR/tests/ui/not_send2.rs:8:26: 8:28}: Ungil` + = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` --> $RUST/core/src/marker.rs | diff --git a/tests/ui/pyclass_probe.rs b/tests/ui/pyclass_probe.rs new file mode 100644 index 00000000000..590af194f6f --- /dev/null +++ b/tests/ui/pyclass_probe.rs @@ -0,0 +1,6 @@ +use pyo3::prelude::*; + +#[pyclass] +pub struct Probe {} + +fn main() {} diff --git a/tests/ui/pyclass_send.stderr b/tests/ui/pyclass_send.stderr index 3ef9591f820..1623a5b2183 100644 --- a/tests/ui/pyclass_send.stderr +++ b/tests/ui/pyclass_send.stderr @@ -4,13 +4,13 @@ error[E0277]: `*mut c_void` cannot be shared between threads safely 5 | struct NotSyncNotSend(*mut c_void); | ^^^^^^^^^^^^^^ `*mut c_void` cannot be shared between threads safely | - = help: within `NotSyncNotSend`, the trait `Sync` is not implemented for `*mut c_void`, which is required by `NotSyncNotSend: Sync` + = help: within `NotSyncNotSend`, the trait `Sync` is not implemented for `*mut c_void` note: required because it appears within the type `NotSyncNotSend` --> tests/ui/pyclass_send.rs:5:8 | 5 | struct NotSyncNotSend(*mut c_void); | ^^^^^^^^^^^^^^ -note: required by a bound in `pyo3::impl_::pyclass::assertions::assert_pyclass_sync` +note: required by a bound in `assert_pyclass_sync` --> src/impl_/pyclass/assertions.rs | | pub const fn assert_pyclass_sync() @@ -25,7 +25,7 @@ error[E0277]: `*mut c_void` cannot be sent between threads safely 4 | #[pyclass] | ^^^^^^^^^^ `*mut c_void` cannot be sent between threads safely | - = help: within `NotSyncNotSend`, the trait `Send` is not implemented for `*mut c_void`, which is required by `SendablePyClass: pyo3::impl_::pyclass::PyClassThreadChecker` + = help: within `NotSyncNotSend`, the trait `Send` is not implemented for `*mut c_void` = help: the trait `pyo3::impl_::pyclass::PyClassThreadChecker` is implemented for `SendablePyClass` note: required because it appears within the type `NotSyncNotSend` --> tests/ui/pyclass_send.rs:5:8 @@ -46,13 +46,13 @@ error[E0277]: `*mut c_void` cannot be shared between threads safely 8 | struct SendNotSync(*mut c_void); | ^^^^^^^^^^^ `*mut c_void` cannot be shared between threads safely | - = help: within `SendNotSync`, the trait `Sync` is not implemented for `*mut c_void`, which is required by `SendNotSync: Sync` + = help: within `SendNotSync`, the trait `Sync` is not implemented for `*mut c_void` note: required because it appears within the type `SendNotSync` --> tests/ui/pyclass_send.rs:8:8 | 8 | struct SendNotSync(*mut c_void); | ^^^^^^^^^^^ -note: required by a bound in `pyo3::impl_::pyclass::assertions::assert_pyclass_sync` +note: required by a bound in `assert_pyclass_sync` --> src/impl_/pyclass/assertions.rs | | pub const fn assert_pyclass_sync() @@ -67,7 +67,7 @@ error[E0277]: `*mut c_void` cannot be sent between threads safely 11 | #[pyclass] | ^^^^^^^^^^ `*mut c_void` cannot be sent between threads safely | - = help: within `SyncNotSend`, the trait `Send` is not implemented for `*mut c_void`, which is required by `SendablePyClass: pyo3::impl_::pyclass::PyClassThreadChecker` + = help: within `SyncNotSend`, the trait `Send` is not implemented for `*mut c_void` = help: the trait `pyo3::impl_::pyclass::PyClassThreadChecker` is implemented for `SendablePyClass` note: required because it appears within the type `SyncNotSend` --> tests/ui/pyclass_send.rs:12:8 @@ -88,7 +88,7 @@ error[E0277]: `*mut c_void` cannot be sent between threads safely 4 | #[pyclass] | ^^^^^^^^^^ `*mut c_void` cannot be sent between threads safely | - = help: within `NotSyncNotSend`, the trait `Send` is not implemented for `*mut c_void`, which is required by `NotSyncNotSend: Send` + = help: within `NotSyncNotSend`, the trait `Send` is not implemented for `*mut c_void` note: required because it appears within the type `NotSyncNotSend` --> tests/ui/pyclass_send.rs:5:8 | @@ -107,7 +107,7 @@ error[E0277]: `*mut c_void` cannot be sent between threads safely 11 | #[pyclass] | ^^^^^^^^^^ `*mut c_void` cannot be sent between threads safely | - = help: within `SyncNotSend`, the trait `Send` is not implemented for `*mut c_void`, which is required by `SyncNotSend: Send` + = help: within `SyncNotSend`, the trait `Send` is not implemented for `*mut c_void` note: required because it appears within the type `SyncNotSend` --> tests/ui/pyclass_send.rs:12:8 | diff --git a/tests/ui/reject_generics.stderr b/tests/ui/reject_generics.stderr index 47999f36275..fa5359eeca5 100644 --- a/tests/ui/reject_generics.stderr +++ b/tests/ui/reject_generics.stderr @@ -1,10 +1,10 @@ -error: #[pyclass] cannot have generic parameters. For an explanation, see https://pyo3.rs/v0.23.3/class.html#no-generic-parameters +error: #[pyclass] cannot have generic parameters. For an explanation, see https://pyo3.rs/v0.23.4/class.html#no-generic-parameters --> tests/ui/reject_generics.rs:4:25 | 4 | struct ClassWithGenerics { | ^ -error: #[pyclass] cannot have lifetime parameters. For an explanation, see https://pyo3.rs/v0.23.3/class.html#no-lifetime-parameters +error: #[pyclass] cannot have lifetime parameters. For an explanation, see https://pyo3.rs/v0.23.4/class.html#no-lifetime-parameters --> tests/ui/reject_generics.rs:9:27 | 9 | struct ClassWithLifetimes<'a> {