chore: use new Bound API

Author: ovr

packages/cubejs-backend-native/Cargo.toml

@@ -34,7 +34,7 @@ log-reroute = "0.1"
 minijinja = { version = "1", features = ["json", "loader"] }
 once_cell = "1.10"
 # python
-pyo3 = { version = "0.22.6", features = ["gil-refs", "py-clone"], optional = true }
+pyo3 = { version = "0.22.6", features = ["py-clone"], optional = true }
 pyo3-async-runtimes = { version = "0.22.0", features = ["tokio-runtime", "attributes"], optional = true }
 serde = { version = "1.0.217", features = ["derive"] }
 serde_json = "1.0.127"

packages/cubejs-backend-native/src/cross/clrepr_python.rs

@@ -2,10 +2,11 @@ use crate::cross::clrepr::CLReprObject;
 use crate::cross::{CLRepr, CLReprObjectKind, StringType};
 use pyo3::exceptions::{PyNotImplementedError, PyTypeError};
 use pyo3::types::{
-    PyBool, PyComplex, PyDate, PyDict, PyFloat, PyFrame, PyFunction, PyInt, PyList, PySequence,
-    PySet, PyString, PyTraceback, PyTuple,
+    PyAnyMethods, PyBool, PyBoolMethods, PyComplex, PyDate, PyDict, PyDictMethods, PyFloat,
+    PyFloatMethods, PyFrame, PyFunction, PyInt, PyList, PyListMethods, PySequence, PySet,
+    PySetMethods, PyString, PyTraceback, PyTuple, PyTupleMethods, PyTypeMethods,
 };
-use pyo3::{Py, PyAny, PyErr, PyObject, Python, ToPyObject};
+use pyo3::{Bound, Py, PyAny, PyErr, PyObject, Python, ToPyObject};
 
 #[derive(Debug, Clone)]
 pub enum PythonRef {
@@ -18,7 +19,7 @@ pub enum PythonRef {
 
 impl CLRepr {
     /// Convert python value to CLRepr
-    pub fn from_python_ref(v: &PyAny) -> Result<Self, PyErr> {
+    pub fn from_python_ref(v: &Bound<'_, PyAny>) -> Result<Self, PyErr> {
         if v.is_none() {
             return Ok(Self::Null);
         }
@@ -30,7 +31,7 @@ impl CLRepr {
                 StringType::Normal
             };
 
-            Self::String(v.to_string(), string_type)
+            Self::String(v.str()?.to_string(), string_type)
         } else if v.get_type().is_subclass_of::<PyBool>()? {
             Self::Bool(v.downcast::<PyBool>()?.is_true())
         } else if v.get_type().is_subclass_of::<PyFloat>()? {
@@ -47,7 +48,7 @@ impl CLRepr {
                 if k.get_type().is_subclass_of::<PyString>()? {
                     let key_str = k.downcast::<PyString>()?;
 
-                    obj.insert(key_str.to_string(), Self::from_python_ref(v)?);
+                    obj.insert(key_str.to_string(), Self::from_python_ref(&v)?);
                 }
             }
 
@@ -57,7 +58,7 @@ impl CLRepr {
             let mut r = Vec::with_capacity(l.len());
 
             for v in l.iter() {
-                r.push(Self::from_python_ref(v)?);
+                r.push(Self::from_python_ref(&v)?);
             }
 
             Self::Array(r)
@@ -66,7 +67,7 @@ impl CLRepr {
             let mut r = Vec::with_capacity(l.len());
 
             for v in l.iter() {
-                r.push(Self::from_python_ref(v)?);
+                r.push(Self::from_python_ref(&v)?);
             }
 
             Self::Array(r)
@@ -75,12 +76,12 @@ impl CLRepr {
             let mut r = Vec::with_capacity(l.len());
 
             for v in l.iter() {
-                r.push(Self::from_python_ref(v)?);
+                r.push(Self::from_python_ref(&v)?);
             }
 
             Self::Tuple(r)
         } else if v.get_type().is_subclass_of::<PyFunction>()? {
-            let fun: Py<PyFunction> = v.downcast::<PyFunction>()?.into();
+            let fun: Py<PyFunction> = v.downcast::<PyFunction>()?.clone().unbind();
 
             Self::PythonRef(PythonRef::PyFunction(fun))
         } else if v.get_type().is_subclass_of::<PyComplex>()? {
@@ -116,12 +117,12 @@ impl CLRepr {
                 )));
             }
 
-            Self::PythonRef(PythonRef::PyObject(v.into()))
+            Self::PythonRef(PythonRef::PyObject(v.clone().unbind()))
         })
     }
 
-    fn into_py_dict_impl(obj: CLReprObject, py: Python) -> Result<&PyDict, PyErr> {
-        let r = PyDict::new(py);
+    fn into_py_dict_impl(obj: CLReprObject, py: Python) -> Result<Bound<'_, PyDict>, PyErr> {
+        let r = PyDict::new_bound(py);
 
         for (k, v) in obj.into_iter() {
             r.set_item(k, Self::into_py_impl(v, py)?)?;
@@ -132,11 +133,11 @@ impl CLRepr {
 
     fn into_py_impl(from: CLRepr, py: Python) -> Result<PyObject, PyErr> {
         Ok(match from {
-            CLRepr::String(v, _) => PyString::new(py, &v).to_object(py),
-            CLRepr::Bool(v) => PyBool::new(py, v).to_object(py),
-            CLRepr::Float(v) => PyFloat::new(py, v).to_object(py),
+            CLRepr::String(v, _) => PyString::new_bound(py, &v).to_object(py),
+            CLRepr::Bool(v) => PyBool::new_bound(py, v).to_object(py),
+            CLRepr::Float(v) => PyFloat::new_bound(py, v).to_object(py),
             CLRepr::Int(v) => {
-                let py_int: &PyInt = unsafe { py.from_owned_ptr(pyo3::ffi::PyLong_FromLong(v)) };
+                let py_int = unsafe { Bound::from_owned_ptr(py, pyo3::ffi::PyLong_FromLong(v)) };
 
                 py_int.to_object(py)
             }
@@ -147,7 +148,7 @@ impl CLRepr {
                     elements.push(Self::into_py_impl(el, py)?);
                 }
 
-                PyList::new(py, elements).to_object(py)
+                PyList::new_bound(py, elements).to_object(py)
             }
             CLRepr::Tuple(arr) => {
                 let mut elements = Vec::with_capacity(arr.len());
@@ -156,7 +157,7 @@ impl CLRepr {
                     elements.push(Self::into_py_impl(el, py)?);
                 }
 
-                PyTuple::new(py, elements).to_object(py)
+                PyTuple::new_bound(py, elements).to_object(py)
             }
             CLRepr::Object(obj) => {
                 let r = Self::into_py_dict_impl(obj, py)?;
@@ -189,7 +190,7 @@ impl CLRepr {
         })
     }
 
-    pub fn into_py_dict(self, py: Python) -> Result<&PyDict, PyErr> {
+    pub fn into_py_dict(self, py: Python) -> Result<Bound<'_, PyDict>, PyErr> {
         Ok(match self {
             CLRepr::Object(obj) => Self::into_py_dict_impl(obj, py)?,
             other => {

packages/cubejs-backend-native/src/python/cube_config.rs

@@ -1,6 +1,7 @@
 use convert_case::{Case, Casing};
 use neon::prelude::*;
-use pyo3::{PyAny, PyResult};
+use pyo3::types::PyAnyMethods;
+use pyo3::{Bound, PyAny, PyResult};
 
 use crate::cross::{CLRepr, CLReprObject, CLReprObjectKind};
 
@@ -69,10 +70,10 @@ impl CubeConfigPy {
         ]
     }
 
-    pub fn attr(&mut self, config_module: &PyAny, key: &str) -> PyResult<()> {
+    pub fn attr(&mut self, config_module: &Bound<'_, PyAny>, key: &str) -> PyResult<()> {
         let v = config_module.getattr(key)?;
         if !v.is_none() {
-            let value = CLRepr::from_python_ref(v)?;
+            let value = CLRepr::from_python_ref(&v)?;
             self.properties.insert(key.to_case(Case::Camel), value);
         };
 

packages/cubejs-backend-native/src/python/entry.rs

@@ -6,18 +6,19 @@ use crate::python::runtime::py_runtime_init;
 use neon::prelude::*;
 use pyo3::exceptions::PyException;
 use pyo3::prelude::*;
-use pyo3::types::{PyDict, PyFunction, PyList, PyString, PyTuple};
+use pyo3::types::{PyAnyMethods, PyDict, PyFunction, PyList, PyString, PyTuple};
 use std::path::Path;
 
 fn extend_sys_path(py: Python, file_name: &String) -> PyResult<()> {
-    let sys_path = py.import("sys")?.getattr("path")?.downcast::<PyList>()?;
+    let binding = py.import_bound("sys")?.getattr("path")?;
+    let sys_path = binding.downcast::<PyList>()?;
 
     let config_dir = Path::new(&file_name)
         .parent()
         .unwrap_or_else(|| Path::new("."));
     let config_dir_str = config_dir.to_str().unwrap_or(".");
 
-    sys_path.insert(0, PyString::new(py, config_dir_str))?;
+    sys_path.insert(0, PyString::new_bound(py, config_dir_str))?;
     Ok(())
 }
 
@@ -40,9 +41,10 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
             env!("CARGO_MANIFEST_DIR"),
             "/python/cube/src/__init__.py"
         ));
-        PyModule::from_code(py, cube_code, "__init__.py", "cube")?;
+        PyModule::from_code_bound(py, cube_code, "__init__.py", "cube")?;
 
-        let config_module = PyModule::from_code(py, &file_content_arg, &options_file_name, "")?;
+        let config_module =
+            PyModule::from_code_bound(py, &file_content_arg, &options_file_name, "")?;
         let settings_py = if config_module.hasattr("config")? {
             config_module.getattr("config")?
         } else {
@@ -60,7 +62,7 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
         let mut cube_conf = CubeConfigPy::new();
 
         for attr_name in cube_conf.get_attrs() {
-            cube_conf.attr(settings_py, attr_name)?;
+            cube_conf.attr(&settings_py, attr_name)?;
         }
 
         Ok(cube_conf)
@@ -90,9 +92,9 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult<JsPromise> {
             env!("CARGO_MANIFEST_DIR"),
             "/python/cube/src/__init__.py"
         ));
-        PyModule::from_code(py, cube_code, "__init__.py", "cube")?;
+        PyModule::from_code_bound(py, cube_code, "__init__.py", "cube")?;
 
-        let model_module = PyModule::from_code(py, &model_content, &model_file_name, "")?;
+        let model_module = PyModule::from_code_bound(py, &model_content, &model_file_name, "")?;
 
         let mut collected_functions = CLReprObject::new(CLReprObjectKind::Object);
         let mut collected_variables = CLReprObject::new(CLReprObjectKind::Object);
@@ -101,48 +103,55 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult<JsPromise> {
         if model_module.hasattr("template")? {
             let template = model_module.getattr("template")?;
 
-            let functions = template.getattr("functions")?.downcast::<PyDict>()?;
+            let functions_attr = template.getattr("functions")?;
+            let functions = functions_attr.downcast::<PyDict>()?;
             for (local_key, local_value) in functions.iter() {
                 if local_value.is_instance_of::<PyFunction>() {
-                    let fun: Py<PyFunction> = local_value.downcast::<PyFunction>()?.into();
+                    let fun: Py<PyFunction> =
+                        local_value.downcast::<PyFunction>()?.clone().unbind();
                     collected_functions.insert(
-                        local_key.to_string(),
+                        local_key.str()?.to_string(),
                         CLRepr::PythonRef(PythonRef::PyExternalFunction(fun)),
                     );
                 }
             }
 
-            let variables = template.getattr("variables")?.downcast::<PyDict>()?;
+            let variables_attr = template.getattr("variables")?;
+            let variables = variables_attr.downcast::<PyDict>()?;
             for (local_key, local_value) in variables.iter() {
-                collected_variables
-                    .insert(local_key.to_string(), CLRepr::from_python_ref(local_value)?);
+                collected_variables.insert(
+                    local_key.str()?.to_string(),
+                    CLRepr::from_python_ref(&local_value)?,
+                );
             }
 
-            let filters = template.getattr("filters")?.downcast::<PyDict>()?;
+            let filters_attr = template.getattr("filters")?;
+            let filters = filters_attr.downcast::<PyDict>()?;
             for (local_key, local_value) in filters.iter() {
-                let fun: Py<PyFunction> = local_value.downcast::<PyFunction>()?.into();
+                let fun: Py<PyFunction> = local_value.downcast::<PyFunction>()?.clone().unbind();
                 collected_filters.insert(
-                    local_key.to_string(),
+                    local_key.str()?.to_string(),
                     CLRepr::PythonRef(PythonRef::PyExternalFunction(fun)),
                 );
             }
         } else {
             // backward compatibility, was used in private preview, not as Public API
             // TODO: Remove after 1.4
-            let inspect_module = py.import("inspect")?;
+            let inspect_module = py.import_bound("inspect")?;
             let args = (model_module, inspect_module.getattr("isfunction")?);
-            let functions_with_names = inspect_module
-                .call_method1("getmembers", args)?
-                .downcast::<PyList>()?;
+            let getmembers_result = inspect_module.call_method1("getmembers", args)?;
+            let functions_with_names = getmembers_result.downcast::<PyList>()?;
 
             for function_details in functions_with_names.iter() {
                 let function_details = function_details.downcast::<PyTuple>()?;
-                let fun_name = function_details.get_item(0)?.downcast::<PyString>()?;
-                let fun = function_details.get_item(1)?.downcast::<PyFunction>()?;
+                let fun_name_item = function_details.get_item(0)?;
+                let fun_name = fun_name_item.downcast::<PyString>()?;
+                let fun_item = function_details.get_item(1)?;
+                let fun = fun_item.downcast::<PyFunction>()?;
 
                 let has_attr = fun.hasattr("cube_context_func")?;
                 if has_attr {
-                    let fun: Py<PyFunction> = fun.into();
+                    let fun: Py<PyFunction> = fun.clone().unbind();
                     collected_functions.insert(
                         fun_name.to_string(),
                         CLRepr::PythonRef(PythonRef::PyExternalFunction(fun)),

packages/cubejs-backend-native/src/python/neon_py.rs

@@ -5,7 +5,7 @@ pub(crate) fn format_python_error(py_err: PyErr) -> String {
     let err = format!("Python error: {}", py_err);
 
     let bt = Python::with_gil(move |py| -> PyResult<Option<String>> {
-        if let Some(trace_back) = py_err.traceback(py) {
+        if let Some(trace_back) = py_err.traceback_bound(py) {
             Ok(Some(trace_back.format()?))
         } else {
             Ok(None)

packages/cubejs-backend-native/src/python/runtime.rs

@@ -96,26 +96,27 @@ impl PyRuntime {
 
         let task_result = Python::with_gil(move |py| -> PyResult<PyScheduledFunResult> {
             let mut prep_tuple = Vec::with_capacity(args.len());
-            let mut py_kwargs = None;
+            let mut kwargs_dict_opt = None;
 
             for arg in args {
                 if arg.is_kwarg() {
-                    py_kwargs = Some(arg.into_py_dict(py)?);
+                    kwargs_dict_opt = Some(arg.into_py_dict(py)?);
                 } else {
                     prep_tuple.push(arg.into_py(py)?);
                 }
             }
 
-            let py_args = PyTuple::new(py, prep_tuple);
-            let call_res = fun.call(py, py_args, py_kwargs)?;
+            let py_args = PyTuple::new_bound(py, prep_tuple);
+            let py_kwargs = kwargs_dict_opt.as_ref();
+            let call_res = fun.call_bound(py, py_args, py_kwargs)?;
 
             let is_coroutine = unsafe { pyo3::ffi::PyCoro_CheckExact(call_res.as_ptr()) == 1 };
             if is_coroutine {
                 let fut = pyo3_async_runtimes::tokio::into_future(call_res.bind(py).clone())?;
                 Ok(PyScheduledFunResult::Poll(Box::pin(fut)))
             } else {
                 Ok(PyScheduledFunResult::Ready(CLRepr::from_python_ref(
-                    call_res.as_ref(py),
+                    &call_res.bind(py),
                 )?))
             }
         });
@@ -155,7 +156,7 @@ impl PyRuntime {
 
                     let res = Python::with_gil(move |py| -> Result<CLRepr, PyErr> {
                         let res = match fut_res {
-                            Ok(r) => CLRepr::from_python_ref(r.as_ref(py)),
+                            Ok(r) => CLRepr::from_python_ref(&r.bind(py)),
                             Err(err) => Err(err),
                         };
 

packages/cubejs-backend-native/src/python/utils.rs

@@ -11,7 +11,7 @@ pub fn python_fn_call_sync(py_fun: &Py<PyFunction>, arguments: Vec<CLRepr>) -> P
             args_tuple.push(arg.into_py(py)?);
         }
 
-        let tuple = PyTuple::new(py, args_tuple);
+        let tuple = PyTuple::new_bound(py, args_tuple);
 
         let call_res = py_fun.call1(py, tuple)?;
 
@@ -21,7 +21,7 @@ pub fn python_fn_call_sync(py_fun: &Py<PyFunction>, arguments: Vec<CLRepr>) -> P
                 "Calling function with async response is not supported",
             ))
         } else {
-            CLRepr::from_python_ref(call_res.as_ref(py))
+            CLRepr::from_python_ref(&call_res.bind(py))
         }
     })
 }
@@ -34,7 +34,7 @@ pub fn python_obj_call_sync(py_fun: &PyObject, arguments: Vec<CLRepr>) -> PyResu
             args_tuple.push(arg.into_py(py)?);
         }
 
-        let tuple = PyTuple::new(py, args_tuple);
+        let tuple = PyTuple::new_bound(py, args_tuple);
 
         let call_res = py_fun.call1(py, tuple)?;
 
@@ -44,7 +44,7 @@ pub fn python_obj_call_sync(py_fun: &PyObject, arguments: Vec<CLRepr>) -> PyResu
                 "Calling object with async response is not supported",
             ))
         } else {
-            CLRepr::from_python_ref(call_res.as_ref(py))
+            CLRepr::from_python_ref(&call_res.bind(py))
         }
     })
 }
@@ -64,7 +64,7 @@ where
             args_tuple.push(arg.into_py(py)?);
         }
 
-        let tuple = PyTuple::new(py, args_tuple);
+        let tuple = PyTuple::new_bound(py, args_tuple);
 
         let call_res = py_fun.call_method1(py, name, tuple)?;
 
@@ -74,7 +74,7 @@ where
                 "Calling object method with async response is not supported",
             ))
         } else {
-            CLRepr::from_python_ref(call_res.as_ref(py))
+            CLRepr::from_python_ref(&call_res.bind(py))
         }
     })
 }