Skip to content

Commit 6a5f70c

Browse files
ovrovr
authored
feat(native): Jinja - support dynamic Python objects (methods, __call__, fields) (#7162)
1 parent a32b619 commit 6a5f70c

File tree

9 files changed

+295
-81
lines changed

9 files changed

+295
-81
lines changed

‎packages/cubejs-backend-native/src/python/cross.rs‎

Lines changed: 89 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@ use neon::prelude::*;
44
use neon::result::Throw;
55
use neon::types::JsDate;
66
use pyo3::exceptions::{PyNotImplementedError, PyTypeError};
7-
use pyo3::types::{PyBool, PyDict, PyFloat, PyFunction, PyInt, PyList, PySet, PyString, PyTuple};
8-
use pyo3::{Py, PyAny, PyErr, PyObject, Python, ToPyObject};
7+
use pyo3::types::{
8+
PyBool, PyComplex, PyDate, PyDict, PyFloat, PyFrame, PyFunction, PyInt, PyList, PySet,
9+
PyString, PyTraceback, PyTuple,
10+
};
11+
use pyo3::{AsPyPointer, Py, PyAny, PyErr, PyObject, Python, ToPyObject};
912
use std::cell::RefCell;
1013
use std::collections::hash_map::{IntoIter, Iter, Keys};
1114
use std::collections::HashMap;
@@ -63,6 +66,7 @@ pub enum CLReprKind {
6366
Array,
6467
Object,
6568
JsFunction,
69+
PyObject,
6670
PyFunction,
6771
PyExternalFunction,
6872
Null,
@@ -82,6 +86,7 @@ pub enum CLRepr {
8286
Array(Vec<CLRepr>),
8387
Object(CLReprObject),
8488
JsFunction(Arc<Root<JsFunction>>),
89+
PyObject(Py<PyAny>),
8590
PyFunction(Py<PyFunction>),
8691
/// Special type to transfer functions through JavaScript
8792
/// In JS it's an external object. It's not the same as Function.
@@ -139,10 +144,7 @@ struct IntoJsContext {
139144

140145
impl CLRepr {
141146
pub fn is_null(&self) -> bool {
142-
match self {
143-
CLRepr::Null => true,
144-
_ => false,
145-
}
147+
matches!(self, CLRepr::Null)
146148
}
147149

148150
pub fn downcast_to_object(self) -> CLReprObject {
@@ -163,6 +165,7 @@ impl CLRepr {
163165
CLRepr::Array(_) => CLReprKind::Array,
164166
CLRepr::Object(_) => CLReprKind::Object,
165167
CLRepr::JsFunction(_) => CLReprKind::JsFunction,
168+
CLRepr::PyObject(_) => CLReprKind::PyObject,
166169
CLRepr::PyFunction(_) => CLReprKind::PyFunction,
167170
CLRepr::PyExternalFunction(_) => CLReprKind::PyExternalFunction,
168171
CLRepr::Null => CLReprKind::Null,
@@ -235,70 +238,90 @@ impl CLRepr {
235238

236239
/// Convert python value to CLRepr
237240
pub fn from_python_ref(v: &PyAny) -> Result<Self, PyErr> {
238-
if !v.is_none() {
239-
return Ok(if v.get_type().is_subclass_of::<PyString>()? {
240-
Self::String(v.to_string())
241-
} else if v.get_type().is_subclass_of::<PyBool>()? {
242-
Self::Bool(v.downcast::<PyBool>()?.is_true())
243-
} else if v.get_type().is_subclass_of::<PyFloat>()? {
244-
let f = v.downcast::<PyFloat>()?;
245-
Self::Float(f.value())
246-
} else if v.get_type().is_subclass_of::<PyInt>()? {
247-
let i: i64 = v.downcast::<PyInt>()?.extract()?;
248-
Self::Int(i)
249-
} else if v.get_type().is_subclass_of::<PyDict>()? {
250-
let d = v.downcast::<PyDict>()?;
251-
let mut obj = CLReprObject::new();
252-
253-
for (k, v) in d.iter() {
254-
if k.get_type().is_subclass_of::<PyString>()? {
255-
let key_str = k.downcast::<PyString>()?;
256-
257-
obj.insert(key_str.to_string(), Self::from_python_ref(v)?);
258-
}
259-
}
241+
if v.is_none() {
242+
return Ok(Self::Null);
243+
}
244+
245+
return Ok(if v.get_type().is_subclass_of::<PyString>()? {
246+
Self::String(v.to_string())
247+
} else if v.get_type().is_subclass_of::<PyBool>()? {
248+
Self::Bool(v.downcast::<PyBool>()?.is_true())
249+
} else if v.get_type().is_subclass_of::<PyFloat>()? {
250+
let f = v.downcast::<PyFloat>()?;
251+
Self::Float(f.value())
252+
} else if v.get_type().is_subclass_of::<PyInt>()? {
253+
let i: i64 = v.downcast::<PyInt>()?.extract()?;
254+
Self::Int(i)
255+
} else if v.get_type().is_subclass_of::<PyDict>()? {
256+
let d = v.downcast::<PyDict>()?;
257+
let mut obj = CLReprObject::new();
260258

261-
Self::Object(obj)
262-
} else if v.get_type().is_subclass_of::<PyList>()? {
263-
let l = v.downcast::<PyList>()?;
264-
let mut r = Vec::with_capacity(l.len());
259+
for (k, v) in d.iter() {
260+
if k.get_type().is_subclass_of::<PyString>()? {
261+
let key_str = k.downcast::<PyString>()?;
265262

266-
for v in l.iter() {
267-
r.push(Self::from_python_ref(v)?);
263+
obj.insert(key_str.to_string(), Self::from_python_ref(v)?);
268264
}
265+
}
269266

270-
Self::Array(r)
271-
} else if v.get_type().is_subclass_of::<PySet>()? {
272-
let l = v.downcast::<PySet>()?;
273-
let mut r = Vec::with_capacity(l.len());
267+
Self::Object(obj)
268+
} else if v.get_type().is_subclass_of::<PyList>()? {
269+
let l = v.downcast::<PyList>()?;
270+
let mut r = Vec::with_capacity(l.len());
274271

275-
for v in l.iter() {
276-
r.push(Self::from_python_ref(v)?);
277-
}
272+
for v in l.iter() {
273+
r.push(Self::from_python_ref(v)?);
274+
}
278275

279-
Self::Array(r)
280-
} else if v.get_type().is_subclass_of::<PyTuple>()? {
281-
let l = v.downcast::<PyTuple>()?;
282-
let mut r = Vec::with_capacity(l.len());
276+
Self::Array(r)
277+
} else if v.get_type().is_subclass_of::<PySet>()? {
278+
let l = v.downcast::<PySet>()?;
279+
let mut r = Vec::with_capacity(l.len());
283280

284-
for v in l.iter() {
285-
r.push(Self::from_python_ref(v)?);
286-
}
281+
for v in l.iter() {
282+
r.push(Self::from_python_ref(v)?);
283+
}
287284

288-
Self::Tuple(r)
289-
} else if v.get_type().is_subclass_of::<PyFunction>()? {
290-
let fun: Py<PyFunction> = v.downcast::<PyFunction>()?.into();
285+
Self::Array(r)
286+
} else if v.get_type().is_subclass_of::<PyTuple>()? {
287+
let l = v.downcast::<PyTuple>()?;
288+
let mut r = Vec::with_capacity(l.len());
291289

292-
Self::PyFunction(fun)
293-
} else {
294-
return Err(PyErr::new::<PyTypeError, _>(format!(
295-
"Unable to represent {} type as CLR from Python",
296-
v.get_type(),
297-
)));
298-
});
299-
}
290+
for v in l.iter() {
291+
r.push(Self::from_python_ref(v)?);
292+
}
293+
294+
Self::Tuple(r)
295+
} else if v.get_type().is_subclass_of::<PyFunction>()? {
296+
let fun: Py<PyFunction> = v.downcast::<PyFunction>()?.into();
297+
298+
Self::PyFunction(fun)
299+
} else if v.get_type().is_subclass_of::<PyComplex>()? {
300+
return Err(PyErr::new::<PyTypeError, _>(
301+
"Unable to represent PyComplex type as CLR from Python".to_string(),
302+
));
303+
} else if v.get_type().is_subclass_of::<PyDate>()? {
304+
return Err(PyErr::new::<PyTypeError, _>(
305+
"Unable to represent PyDate type as CLR from Python".to_string(),
306+
));
307+
} else if v.get_type().is_subclass_of::<PyFrame>()? {
308+
return Err(PyErr::new::<PyTypeError, _>(
309+
"Unable to represent PyFrame type as CLR from Python".to_string(),
310+
));
311+
} else if v.get_type().is_subclass_of::<PyTraceback>()? {
312+
return Err(PyErr::new::<PyTypeError, _>(
313+
"Unable to represent PyTraceback type as CLR from Python".to_string(),
314+
));
315+
} else {
316+
let is_sequence = unsafe { pyo3::ffi::PySequence_Check(v.as_ptr()) == 1 };
317+
if is_sequence {
318+
return Err(PyErr::new::<PyTypeError, _>(
319+
"Unable to represent PyTraceback type as CLR from Python".to_string(),
320+
));
321+
}
300322

301-
Ok(Self::Null)
323+
Self::PyObject(v.into())
324+
});
302325
}
303326

304327
fn into_js_impl<'a, C: Context<'a>>(
@@ -373,6 +396,7 @@ impl CLRepr {
373396

374397
unwrapper_fun.into_inner(cx).upcast()
375398
}
399+
CLRepr::PyObject(_) => return cx.throw_error("Unable to represent PyObject in JS"),
376400
})
377401
}
378402

@@ -439,6 +463,11 @@ impl CLRepr {
439463
"Unable to represent PyFunction in Python",
440464
))
441465
}
466+
CLRepr::PyObject(_) => {
467+
return Err(PyErr::new::<PyNotImplementedError, _>(
468+
"Unable to represent PyObject in Python",
469+
))
470+
}
442471
})
443472
}
444473

‎packages/cubejs-backend-native/src/python/entry.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult<JsPromise> {
6868
env!("CARGO_MANIFEST_DIR"),
6969
"/python/cube/src/__init__.py"
7070
));
71-
PyModule::from_code(py, &cube_code, "__init__.py", "cube")?;
71+
PyModule::from_code(py, cube_code, "__init__.py", "cube")?;
7272

7373
let model_module = PyModule::from_code(py, &model_content, &model_file_name, "")?;
7474
let mut collected_functions = CLReprObject::new();

‎packages/cubejs-backend-native/src/python/runtime.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ impl PyRuntime {
117117
match callback {
118118
PyScheduledCallback::NodeDeferred(deferred) => {
119119
deferred.settle_with(
120-
&js_channel,
120+
js_channel,
121121
move |mut cx| -> NeonResult<Handle<JsError>> {
122122
cx.throw_error(format!("Python error: {}", err))
123123
},
@@ -175,7 +175,7 @@ impl PyRuntime {
175175
}
176176
PyScheduledFunResult::Ready(r) => match callback {
177177
PyScheduledCallback::NodeDeferred(deferred) => {
178-
deferred.settle_with(&js_channel, |mut cx| r.into_js(&mut cx));
178+
deferred.settle_with(js_channel, |mut cx| r.into_js(&mut cx));
179179
}
180180
PyScheduledCallback::Channel(chan) => {
181181
if chan.send(Ok(r)).is_err() {
@@ -237,8 +237,8 @@ pub fn py_runtime_init<'a, C: Context<'a>>(
237237
// it's safe to unwrap
238238
pyo3_asyncio::tokio::init_with_runtime(runtime).unwrap();
239239

240-
if let Err(_) = PY_RUNTIME.set(PyRuntime::new(channel)) {
241-
cx.throw_error(format!("Error on setting PyRuntime"))
240+
if PY_RUNTIME.set(PyRuntime::new(channel)).is_err() {
241+
cx.throw_error("Error on setting PyRuntime")
242242
} else {
243243
Ok(())
244244
}

0 commit comments

Comments
 (0)