refactor: extract py_utils as a separate package (#1299)

Author: georgeh0

Cargo.lock

@@ -1,5 +1,5 @@
 [workspace]
-members = ["rust/cocoindex", "rust/utils"]
+members = ["rust/*"]
 resolver = "2"
 
 [workspace.package]

Cargo.toml

@@ -14,7 +14,8 @@ default = ["legacy-states-v0"]
 legacy-states-v0 = []
 
 [dependencies]
-cocoindex-utils = { path = "../utils" }
+cocoindex_utils = { path = "../utils" }
+cocoindex_py_utils = { path = "../py_utils" }
 
 pyo3 = { workspace = true }
 pythonize = { workspace = true }

rust/cocoindex/Cargo.toml

@@ -1,15 +1,16 @@
+use crate::prelude::*;
+
 use std::time::SystemTime;
 
 use crate::base::{schema::*, spec::IndexOptions, value::*};
-use crate::prelude::*;
 use crate::setup;
 use chrono::TimeZone;
 use serde::Serialize;
 
 pub struct FlowInstanceContext {
     pub flow_instance_name: String,
     pub auth_registry: Arc<AuthRegistry>,
-    pub py_exec_ctx: Option<Arc<crate::py::PythonExecutionContext>>,
+    pub py_exec_ctx: Option<Arc<py_utils::PythonExecutionContext>>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]

rust/cocoindex/src/ops/interface.rs

@@ -1,4 +1,4 @@
-use crate::{ops::sdk::BatchedFunctionExecutor, prelude::*, py::future::from_py_future};
+use crate::{ops::sdk::BatchedFunctionExecutor, prelude::*};
 
 use pyo3::{
     Bound, IntoPyObjectExt, Py, PyAny, Python, pyclass, pymethods,
@@ -13,6 +13,7 @@ use crate::{
     py::{self, ToResultWithPyTrace},
 };
 use anyhow::{Result, anyhow};
+use py_utils::from_py_future;
 
 #[pyclass(name = "OpArgSchema")]
 pub struct PyOpArgSchema {

rust/cocoindex/src/ops/py_factory.rs

@@ -35,3 +35,5 @@ pub(crate) use async_stream::{stream, try_stream};
 pub(crate) use log::{debug, error, info, trace, warn};
 
 pub(crate) use derivative::Derivative;
+
+pub(crate) use cocoindex_py_utils as py_utils;

rust/cocoindex/src/prelude.rs

@@ -9,52 +9,8 @@ use pyo3::types::PyAny;
 use pyo3::types::{PyList, PyTuple};
 use pyo3::{exceptions::PyException, prelude::*};
 use pythonize::{depythonize, pythonize};
-use serde::de::DeserializeOwned;
-use std::ops::Deref;
 
-use super::{AnyhowIntoPyResult, IntoPyResult};
-
-#[derive(Debug)]
-pub struct Pythonized<T>(pub T);
-
-impl<'py, T: DeserializeOwned> FromPyObject<'py> for Pythonized<T> {
-    fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
-        Ok(Pythonized(depythonize(obj).into_py_result()?))
-    }
-}
-
-impl<'py, T: Serialize> IntoPyObject<'py> for &Pythonized<T> {
-    type Target = PyAny;
-    type Output = Bound<'py, PyAny>;
-    type Error = PyErr;
-
-    fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
-        pythonize(py, &self.0).into_py_result()
-    }
-}
-
-impl<'py, T: Serialize> IntoPyObject<'py> for Pythonized<T> {
-    type Target = PyAny;
-    type Output = Bound<'py, PyAny>;
-    type Error = PyErr;
-
-    fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
-        (&self).into_pyobject(py)
-    }
-}
-
-impl<T> Pythonized<T> {
-    pub fn into_inner(self) -> T {
-        self.0
-    }
-}
-
-impl<T> Deref for Pythonized<T> {
-    type Target = T;
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
+use py_utils::{AnyhowIntoPyResult, IntoPyResult};
 
 fn basic_value_to_py_object<'py>(
     py: Python<'py>,

rust/cocoindex/src/py/convert.rs

@@ -13,90 +13,14 @@ use crate::service::query_handler::QueryHandlerSpec;
 use crate::settings::Settings;
 use crate::setup::{self};
 use pyo3::IntoPyObjectExt;
-use pyo3::types::{PyDict, PyModule, PyString};
-use pyo3::{exceptions::PyException, prelude::*};
+use pyo3::prelude::*;
+use pyo3::types::PyModule;
 use pyo3_async_runtimes::tokio::future_into_py;
-use std::fmt::Write;
 use std::sync::Arc;
 
 mod convert;
 pub(crate) use convert::*;
-pub mod future;
-
-pub struct PythonExecutionContext {
-    pub event_loop: Py<PyAny>,
-}
-
-impl PythonExecutionContext {
-    pub fn new(_py: Python<'_>, event_loop: Py<PyAny>) -> Self {
-        Self { event_loop }
-    }
-}
-
-pub trait ToResultWithPyTrace<T> {
-    fn to_result_with_py_trace(self, py: Python<'_>) -> anyhow::Result<T>;
-}
-
-impl<T> ToResultWithPyTrace<T> for Result<T, PyErr> {
-    fn to_result_with_py_trace(self, py: Python<'_>) -> anyhow::Result<T> {
-        match self {
-            Ok(value) => Ok(value),
-            Err(err) => {
-                // Attempt to render a full Python-style traceback including cause/context chain
-                let full_trace: PyResult<String> = (|| {
-                    let exc = err.value(py);
-                    let traceback = PyModule::import(py, "traceback")?;
-                    let tbe_class = traceback.getattr("TracebackException")?;
-                    let tbe = tbe_class.call_method1("from_exception", (exc,))?;
-                    let kwargs = PyDict::new(py);
-                    kwargs.set_item("chain", true)?;
-                    let lines = tbe.call_method("format", (), Some(&kwargs))?;
-                    let joined = PyString::new(py, "").call_method1("join", (lines,))?;
-                    joined.extract::<String>()
-                })();
-
-                let err_str = match full_trace {
-                    Ok(trace) => format!("Error calling Python function:\n{trace}"),
-                    Err(_) => {
-                        // Fallback: include the PyErr display and available traceback formatting
-                        let mut s = format!("Error calling Python function: {err}");
-                        if let Some(tb) = err.traceback(py) {
-                            write!(&mut s, "\n{}", tb.format()?).ok();
-                        }
-                        s
-                    }
-                };
-
-                Err(anyhow::anyhow!(err_str))
-            }
-        }
-    }
-}
-pub trait IntoPyResult<T> {
-    fn into_py_result(self) -> PyResult<T>;
-}
-
-impl<T, E: std::error::Error> IntoPyResult<T> for Result<T, E> {
-    fn into_py_result(self) -> PyResult<T> {
-        match self {
-            Ok(value) => Ok(value),
-            Err(err) => Err(PyException::new_err(format!("{err:?}"))),
-        }
-    }
-}
-
-pub trait AnyhowIntoPyResult<T> {
-    fn into_py_result(self) -> PyResult<T>;
-}
-
-impl<T> AnyhowIntoPyResult<T> for anyhow::Result<T> {
-    fn into_py_result(self) -> PyResult<T> {
-        match self {
-            Ok(value) => Ok(value),
-            Err(err) => Err(PyException::new_err(format!("{err:?}"))),
-        }
-    }
-}
+pub(crate) use py_utils::*;
 
 #[pyfunction]
 fn set_settings_fn(get_settings_fn: Py<PyAny>) -> PyResult<()> {
@@ -480,7 +404,7 @@ impl Flow {
                     let task_locals = pyo3_async_runtimes::TaskLocals::new(
                         py_exec_ctx.event_loop.bind(py).clone(),
                     );
-                    Ok(future::from_py_future(
+                    Ok(py_utils::from_py_future(
                         py,
                         &task_locals,
                         result_coro.into_bound(py),

rust/cocoindex/src/py/mod.rs

@@ -0,0 +1,17 @@
+[package]
+name = "cocoindex_py_utils"
+version = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+license = { workspace = true }
+
+[dependencies]
+cocoindex_utils = { path = "../utils" }
+
+anyhow = { workspace = true }
+futures = { workspace = true }
+pyo3 = { workspace = true }
+pyo3-async-runtimes = { workspace = true }
+pythonize = { workspace = true }
+serde = { workspace = true }
+log = { workspace = true }

rust/py_utils/Cargo.toml

@@ -0,0 +1,48 @@
+use pyo3::prelude::*;
+use pythonize::{depythonize, pythonize};
+use serde::{Serialize, de::DeserializeOwned};
+use std::ops::Deref;
+
+use crate::error::IntoPyResult;
+
+#[derive(Debug)]
+pub struct Pythonized<T>(pub T);
+
+impl<'py, T: DeserializeOwned> FromPyObject<'py> for Pythonized<T> {
+    fn extract_bound(obj: &Bound<'py, PyAny>) -> PyResult<Self> {
+        Ok(Pythonized(depythonize(obj).into_py_result()?))
+    }
+}
+
+impl<'py, T: Serialize> IntoPyObject<'py> for &Pythonized<T> {
+    type Target = PyAny;
+    type Output = Bound<'py, PyAny>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
+        pythonize(py, &self.0).into_py_result()
+    }
+}
+
+impl<'py, T: Serialize> IntoPyObject<'py> for Pythonized<T> {
+    type Target = PyAny;
+    type Output = Bound<'py, PyAny>;
+    type Error = PyErr;
+
+    fn into_pyobject(self, py: Python<'py>) -> PyResult<Self::Output> {
+        (&self).into_pyobject(py)
+    }
+}
+
+impl<T> Pythonized<T> {
+    pub fn into_inner(self) -> T {
+        self.0
+    }
+}
+
+impl<T> Deref for Pythonized<T> {
+    type Target = T;
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}