Update to PyO3 0.28 to automatically disable GIL (#1948)

* Update to PyO3 0.28 to automatically disable GIL

* Use from_py_object for all PyClasses that derive Clone

* Update PyO3 to 0.28.1

* Update to PyO3 0.28.2

* Update dev-dependencies as well

Author: ngoldbaum

bindings/python/Cargo.toml

@@ -14,22 +14,22 @@ serde = { version = "1.0", features = ["rc", "derive"] }
 serde_json = "1.0"
 libc = "0.2"
 env_logger = "0.11"
-pyo3 = { version = "0.27.2",default-features = false, features = ["abi3", "abi3-py39", "py-clone", "experimental-inspect"] }
-pyo3-async-runtimes = { version = "0.27.0", features = ["tokio-runtime"] }
+pyo3 = { version = "0.28.2",default-features = false, features = ["abi3", "abi3-py39", "py-clone", "experimental-inspect"] }
+pyo3-async-runtimes = { version = "0.28.0", features = ["tokio-runtime"] }
 tokio = { version = "1.47.1", features = ["rt", "rt-multi-thread", "macros", "signal"] }
 once_cell = "1.19.0"
-numpy = "0.27"
+numpy = "0.28"
 ndarray = "0.16"
 itertools = "0.14"
 ahash = { version = "0.8.11", features = ["serde"] }
-pyo3-ffi = "0.27.2"
+pyo3-ffi = "0.28.2"
 
 [dependencies.tokenizers]
 path = "../../tokenizers"
 
 [dev-dependencies]
 tempfile = "3.10"
-pyo3 = { version = "0.27.2", features = ["auto-initialize"] }
+pyo3 = { version = "0.28.2", features = ["auto-initialize"] }
 
 
 [features]

bindings/python/src/decoders.rs

@@ -28,7 +28,13 @@ use super::error::ToPyResult;
 ///
 /// This class is not supposed to be instantiated directly. Instead, any implementation of
 /// a Decoder will return an instance of this class when instantiated.
-#[pyclass(dict, module = "tokenizers.decoders", name = "Decoder", subclass)]
+#[pyclass(
+    dict,
+    module = "tokenizers.decoders",
+    name = "Decoder",
+    subclass,
+    from_py_object
+)]
 #[derive(Clone, Deserialize, Serialize)]
 #[serde(transparent)]
 pub struct PyDecoder {
@@ -610,7 +616,7 @@ pub mod decoders {
 
 /// Class needed for streaming decode
 ///
-#[pyclass(module = "tokenizers.decoders", name = "DecodeStream")]
+#[pyclass(module = "tokenizers.decoders", name = "DecodeStream", from_py_object)]
 #[derive(Clone)]
 pub struct PyDecodeStream {
     /// Regular decode option that is kept throughout.

bindings/python/src/models.rs

@@ -25,7 +25,7 @@ use super::error::{deprecation_warning, ToPyResult};
 /// will contain and manage the learned vocabulary.
 ///
 /// This class cannot be constructed directly. Please use one of the concrete models.
-#[pyclass(module = "tokenizers.models", name = "Model", subclass)]
+#[pyclass(module = "tokenizers.models", name = "Model", subclass, from_py_object)]
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(transparent)]
 pub struct PyModel {

bindings/python/src/normalizers.rs

@@ -41,7 +41,13 @@ impl PyNormalizedStringMut<'_> {
 ///
 /// This class is not supposed to be instantiated directly. Instead, any implementation of a
 /// Normalizer will return an instance of this class when instantiated.
-#[pyclass(dict, module = "tokenizers.normalizers", name = "Normalizer", subclass)]
+#[pyclass(
+    dict,
+    module = "tokenizers.normalizers",
+    name = "Normalizer",
+    subclass,
+    from_py_object
+)]
 #[derive(Clone, Debug, Serialize, Deserialize)]
 #[serde(transparent)]
 pub struct PyNormalizer {

bindings/python/src/pre_tokenizers.rs

@@ -34,7 +34,8 @@ use super::utils::*;
     dict,
     module = "tokenizers.pre_tokenizers",
     name = "PreTokenizer",
-    subclass
+    subclass,
+    from_py_object
 )]
 #[derive(Clone, Serialize, Deserialize)]
 #[serde(transparent)]

bindings/python/src/processors.rs

@@ -29,7 +29,8 @@ use tokenizers as tk;
     dict,
     module = "tokenizers.processors",
     name = "PostProcessor",
-    subclass
+    subclass,
+    from_py_object
 )]
 #[derive(Clone, Deserialize, Serialize)]
 #[serde(transparent)]

bindings/python/src/token.rs

@@ -1,7 +1,7 @@
 use pyo3::prelude::*;
 use tk::Token;
 
-#[pyclass(module = "tokenizers", name = "Token", frozen)]
+#[pyclass(module = "tokenizers", name = "Token", frozen, from_py_object)]
 #[derive(Clone)]
 pub struct PyToken {
     token: Token,

bindings/python/src/tokenizer.rs

@@ -482,7 +482,7 @@ type Tokenizer = TokenizerImpl<PyModel, PyNormalizer, PyPreTokenizer, PyPostProc
 ///     model (:class:`~tokenizers.models.Model`):
 ///         The core algorithm that this :obj:`Tokenizer` should be using.
 ///
-#[pyclass(dict, module = "tokenizers", name = "Tokenizer")]
+#[pyclass(dict, module = "tokenizers", name = "Tokenizer", from_py_object)]
 #[derive(Clone, Serialize)]
 #[serde(transparent)]
 pub struct PyTokenizer {

bindings/python/src/trainers.rs

@@ -14,7 +14,12 @@ use tokenizers as tk;
 ///
 /// This class is not supposed to be instantiated directly. Instead, any implementation of a
 /// Trainer will return an instance of this class when instantiated.
-#[pyclass(module = "tokenizers.trainers", name = "Trainer", subclass)]
+#[pyclass(
+    module = "tokenizers.trainers",
+    name = "Trainer",
+    subclass,
+    from_py_object
+)]
 #[derive(Clone, Deserialize, Serialize)]
 #[serde(transparent)]
 pub struct PyTrainer {

bindings/python/src/utils/normalization.rs

@@ -196,7 +196,7 @@ fn slice(
 /// Args:
 ///     sequence: str:
 ///         The string sequence used to initialize this NormalizedString
-#[pyclass(module = "tokenizers", name = "NormalizedString")]
+#[pyclass(module = "tokenizers", name = "NormalizedString", from_py_object)]
 #[derive(Clone)]
 pub struct PyNormalizedString {
     pub(crate) normalized: NormalizedString,
@@ -388,7 +388,7 @@ impl From<PyNormalizedString> for NormalizedString {
     }
 }
 
-#[pyclass(module = "tokenizers", name = "NormalizedStringRefMut")]
+#[pyclass(module = "tokenizers", name = "NormalizedStringRefMut", from_py_object)]
 #[derive(Clone)]
 pub struct PyNormalizedStringRefMut {
     inner: RefMutContainer<NormalizedString>,