Merge branch 'main' into faer_compat

Author: davidhewitt

.github/workflows/ci.yml

@@ -274,6 +274,7 @@ jobs:
           import toml
           cargo_toml = toml.load("Cargo.toml")
           cargo_toml["dependencies"]["ndarray"] = "0.15.6"
+          cargo_toml["dependencies"]["once_cell"] = "1.20.3"
           with open("Cargo.toml", "w") as f:
             toml.dump(cargo_toml, f)
         working-directory: examples/simple
@@ -290,6 +291,8 @@ jobs:
             pkg_id = pkg["name"] + ":" + pkg["version"]
             if pkg["name"] == "ndarray" and pkg["version"] != "0.15.6":
               subprocess.run(["cargo", "update", "--package", pkg_id, "--precise", "0.15.6"], check=True)
+            elif pkg["name"] == "once_cell" and pkg["version"] != "1.20.3":
+              subprocess.run(["cargo", "update", "--package", pkg_id, "--precise", "1.20.3"], check=True)
         working-directory: examples/simple
         shell: python
       - name: Test example

Cargo.toml

@@ -23,18 +23,18 @@ num-complex = ">= 0.2, < 0.5"
 num-integer = "0.1"
 num-traits = "0.2"
 ndarray = ">= 0.15, < 0.17"
-pyo3 = { version = "0.23.4", default-features = false, features = ["macros"] }
+pyo3 = { version = "0.24", default-features = false, features = ["macros"] }
 rustc-hash = "2.0"
 
 [features]
 faer = ["dep:faer"]
 
 [dev-dependencies]
-pyo3 = { version = "0.23.3", default-features = false, features = ["auto-initialize"] }
+pyo3 = { version = "0.24", default-features = false, features = ["auto-initialize"] }
 nalgebra = { version = ">=0.30, <0.34", default-features = false, features = ["std"] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.23.1", features = ["resolve-config"] }
+pyo3-build-config = { version = "0.24", features = ["resolve-config"] }
 
 [package.metadata.docs.rs]
 all-features = true

examples/linalg/Cargo.lock

@@ -9,7 +9,7 @@ name = "rust_linalg"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.23.3", features = ["extension-module"] }
+pyo3 = { version = "0.24.0", features = ["extension-module"] }
 numpy = { path = "../.." }
 ndarray-linalg = { version = "0.14.1", features = ["openblas-system"] }
 

examples/linalg/Cargo.toml

@@ -9,7 +9,7 @@ name = "rust_parallel"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.23.0", features = ["extension-module", "multiple-pymethods"] }
+pyo3 = { version = "0.24.0", features = ["extension-module", "multiple-pymethods"] }
 numpy = { path = "../.." }
 ndarray = { version = "0.16", features = ["rayon", "blas"] }
 blas-src = { version = "0.8", features = ["openblas"] }

examples/parallel/Cargo.toml

@@ -9,7 +9,7 @@ name = "rust_ext"
 crate-type = ["cdylib"]
 
 [dependencies]
-pyo3 = { version = "0.23.0", features = ["extension-module", "abi3-py37"] }
+pyo3 = { version = "0.24.0", features = ["extension-module", "abi3-py37"] }
 numpy = { path = "../.." }
 
 [workspace]

examples/simple/Cargo.toml

@@ -60,14 +60,14 @@ use std::hash::Hash;
 use std::marker::PhantomData;
 use std::sync::Mutex;
 
+use pyo3::sync::MutexExt;
 use pyo3::{Bound, Py, Python};
 use rustc_hash::FxHashMap;
 
 use crate::dtype::{clone_methods_impl, Element, PyArrayDescr, PyArrayDescrMethods};
 use crate::npyffi::{
     PyArray_DatetimeDTypeMetaData, PyDataType_C_METADATA, NPY_DATETIMEUNIT, NPY_TYPES,
 };
-use crate::ThreadStateGuard;
 
 /// Represents the [datetime units][datetime-units] supported by NumPy
 ///
@@ -224,13 +224,10 @@ impl TypeDescriptors {
 
     #[allow(clippy::wrong_self_convention)]
     fn from_unit<'py>(&self, py: Python<'py>, unit: NPY_DATETIMEUNIT) -> Bound<'py, PyArrayDescr> {
-        // Detach from the runtime to avoid deadlocking on acquiring the mutex.
-        let ts_guard = ThreadStateGuard::new();
-
-        let mut dtypes = self.dtypes.lock().expect("dtype cache poisoned");
-
-        // Now we hold the mutex so it's safe to re-attach to the runtime.
-        drop(ts_guard);
+        let mut dtypes = self
+            .dtypes
+            .lock_py_attached(py)
+            .expect("dtype cache poisoned");
 
         let dtype = match dtypes.get_or_insert_with(Default::default).entry(unit) {
             Entry::Occupied(entry) => entry.into_mut(),

src/datetime.rs

@@ -149,23 +149,6 @@ mod doctest {
 #[inline(always)]
 fn cold() {}
 
-/// An RAII guard for avoiding deadlocks with the GIL or other global
-/// synchronization events in the Python runtime
-// FIXME create a proper MutexExt trait that handles poisoning and upstream to PyO3
-struct ThreadStateGuard(*mut pyo3::ffi::PyThreadState);
-
-impl ThreadStateGuard {
-    fn new() -> ThreadStateGuard {
-        ThreadStateGuard(unsafe { pyo3::ffi::PyEval_SaveThread() })
-    }
-}
-
-impl Drop for ThreadStateGuard {
-    fn drop(&mut self) {
-        unsafe { pyo3::ffi::PyEval_RestoreThread(self.0) };
-    }
-}
-
 /// Create a [`PyArray`] with one, two or three dimensions.
 ///
 /// This macro is backed by [`ndarray::array`].

src/lib.rs

@@ -10,6 +10,7 @@ use std::os::raw::c_char;
 use std::str;
 use std::sync::Mutex;
 
+use pyo3::sync::MutexExt;
 use pyo3::{
     ffi::{Py_UCS1, Py_UCS4},
     Bound, Py, Python,
@@ -19,7 +20,6 @@ use rustc_hash::FxHashMap;
 use crate::dtype::{clone_methods_impl, Element, PyArrayDescr, PyArrayDescrMethods};
 use crate::npyffi::PyDataType_SET_ELSIZE;
 use crate::npyffi::NPY_TYPES;
-use crate::ThreadStateGuard;
 
 /// A newtype wrapper around [`[u8; N]`][Py_UCS1] to handle [`byte` scalars][numpy-bytes] while satisfying coherence.
 ///
@@ -179,13 +179,10 @@ impl TypeDescriptors {
         byteorder: c_char,
         size: usize,
     ) -> Bound<'py, PyArrayDescr> {
-        // Detach from the runtime to avoid deadlocking on acquiring the mutex.
-        let ts_guard = ThreadStateGuard::new();
-
-        let mut dtypes = self.dtypes.lock().expect("dtype cache poisoned");
-
-        // Now we hold the mutex so it's safe to re-attach to the runtime.
-        drop(ts_guard);
+        let mut dtypes = self
+            .dtypes
+            .lock_py_attached(py)
+            .expect("dtype cache poisoned");
 
         let dtype = match dtypes.get_or_insert_with(Default::default).entry(size) {
             Entry::Occupied(entry) => entry.into_mut(),