Skip to content

Commit e2569a9

Browse files
ovrovr
authored
feat(native): Jinja - support fields reflection for PyObject (#7312)
1 parent 792e265 commit e2569a9

File tree

12 files changed

+117
-81
lines changed

12 files changed

+117
-81
lines changed

‎packages/cubejs-backend-native/src/cross/clrepr_python.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use pyo3::{AsPyPointer, Py, PyAny, PyErr, PyObject, Python, ToPyObject};
99

1010
#[derive(Debug, Clone)]
1111
pub enum PythonRef {
12-
PyObject(Py<PyAny>),
12+
PyObject(PyObject),
1313
PyFunction(Py<PyFunction>),
1414
/// Special type to transfer functions through JavaScript
1515
/// In JS it's an external object. It's not the same as Function.

‎packages/cubejs-backend-native/src/template/entry.rs‎

Lines changed: 2 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,70 +1,16 @@
11
use crate::cross::*;
22
use crate::template::mj_value::*;
3+
use crate::template::neon::NeonMiniJinjaContext;
34
use crate::utils::bind_method;
5+
46
use log::trace;
57
use minijinja as mj;
6-
use neon::context::Context;
78
use neon::prelude::*;
89
use std::cell::RefCell;
9-
use std::error::Error;
1010

1111
#[cfg(feature = "python")]
1212
use pyo3::{exceptions::PyNotImplementedError, prelude::*, types::PyTuple, AsPyPointer};
1313

14-
trait NeonMiniJinjaContext {
15-
fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T>;
16-
}
17-
18-
impl<'a> NeonMiniJinjaContext for FunctionContext<'a> {
19-
fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T> {
20-
let codeblock = if let Some(source) = err.template_source() {
21-
let lines: Vec<_> = source.lines().enumerate().collect();
22-
let idx = err.line().unwrap_or(1).saturating_sub(1);
23-
let skip = idx.saturating_sub(3);
24-
25-
let pre = lines.iter().skip(skip).take(3.min(idx)).collect::<Vec<_>>();
26-
let post = lines.iter().skip(idx + 1).take(3).collect::<Vec<_>>();
27-
28-
let mut content = "".to_string();
29-
30-
for (idx, line) in pre {
31-
content += &format!("{:>4} | {}\r\n", idx + 1, line);
32-
}
33-
34-
content += &format!("{:>4} > {}\r\n", idx + 1, lines[idx].1);
35-
36-
if let Some(_span) = err.range() {
37-
// TODO(ovr): improve
38-
content += &format!(
39-
" i {}{} {}\r\n",
40-
" ".repeat(0),
41-
"^".repeat(24),
42-
err.kind(),
43-
);
44-
} else {
45-
content += &format!(" | {}\r\n", "^".repeat(24));
46-
}
47-
48-
for (idx, line) in post {
49-
content += &format!("{:>4} | {}\r\n", idx + 1, line);
50-
}
51-
52-
format!("{}\r\n{}\r\n{}", "-".repeat(79), content, "-".repeat(79))
53-
} else {
54-
"".to_string()
55-
};
56-
57-
if let Some(next_err) = err.source() {
58-
self.throw_error(format!(
59-
"{} caused by: {:#}\r\n{}",
60-
err, next_err, codeblock
61-
))
62-
} else {
63-
self.throw_error(format!("{}\r\n{}", err, codeblock))
64-
}
65-
}
66-
}
67-
6814
struct JinjaEngine {
6915
inner: mj::Environment<'static>,
7016
}

‎packages/cubejs-backend-native/src/template/mj_value/python.rs‎

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ use minijinja as mj;
77
use minijinja::value as mjv;
88
use minijinja::value::{Object, ObjectKind, StructObject, Value};
99
use pyo3::exceptions::PyNotImplementedError;
10-
use pyo3::types::{PyFunction, PyTuple};
11-
use pyo3::{AsPyPointer, Py, PyAny, PyErr, PyResult, Python};
10+
use pyo3::types::{PyDict, PyFunction, PyTuple};
11+
use pyo3::{AsPyPointer, Py, PyErr, PyObject, PyResult, Python};
1212
use std::convert::TryInto;
1313
use std::sync::Arc;
1414

@@ -97,7 +97,7 @@ pub fn from_minijinja_value(from: &mjv::Value) -> Result<CLRepr, mj::Error> {
9797
}
9898

9999
pub struct JinjaPythonObject {
100-
pub(crate) inner: Py<PyAny>,
100+
pub(crate) inner: PyObject,
101101
}
102102

103103
impl std::fmt::Debug for JinjaPythonObject {
@@ -222,20 +222,25 @@ impl StructObject for JinjaPythonObject {
222222
}
223223

224224
fn fields(&self) -> Vec<Arc<str>> {
225-
// TODO(ovr): Should we enable it? dump fn?
226-
// let obj_ref = &self.inner;
227-
//
228-
// Python::with_gil(|py| {
229-
// let mut fields = vec![];
230-
//
231-
// for key in obj_ref.as_ref(py).keys() {
232-
// fields.push(key.to_string().into());
233-
// }
234-
//
235-
// fields
236-
// })
237-
238-
vec![]
225+
let obj_ref = &self.inner as &PyObject;
226+
227+
Python::with_gil(|py| {
228+
let mut fields: Vec<Arc<str>> = vec![];
229+
230+
match obj_ref.downcast::<PyDict>(py) {
231+
Ok(dict_ref) => {
232+
for key in dict_ref.keys() {
233+
fields.push(key.to_string().into());
234+
}
235+
}
236+
Err(err) => {
237+
#[cfg(debug_assertions)]
238+
log::trace!("Unable to extract PyDict: {:?}", err)
239+
}
240+
}
241+
242+
fields
243+
})
239244
}
240245
}
241246

‎packages/cubejs-backend-native/src/template/mod.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
mod entry;
22
mod mj_value;
3+
mod neon;
34

45
pub use entry::template_register_module;

‎packages/cubejs-backend-native/src/template/neon.rs‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
use minijinja as mj;
2+
use std::error::Error;
3+
4+
use neon::prelude::*;
5+
6+
pub(crate) trait NeonMiniJinjaContext<'a>: Context<'a> {
7+
fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T> {
8+
let codeblock = if let Some(source) = err.template_source() {
9+
let lines: Vec<_> = source.lines().enumerate().collect();
10+
let idx = err.line().unwrap_or(1).saturating_sub(1);
11+
let skip = idx.saturating_sub(3);
12+
13+
let pre = lines.iter().skip(skip).take(3.min(idx)).collect::<Vec<_>>();
14+
let post = lines.iter().skip(idx + 1).take(3).collect::<Vec<_>>();
15+
16+
let mut content = "".to_string();
17+
18+
for (idx, line) in pre {
19+
content += &format!("{:>4} | {}\r\n", idx + 1, line);
20+
}
21+
22+
content += &format!("{:>4} > {}\r\n", idx + 1, lines[idx].1);
23+
24+
if let Some(_span) = err.range() {
25+
// TODO(ovr): improve
26+
content += &format!(
27+
" i {}{} {}\r\n",
28+
" ".repeat(0),
29+
"^".repeat(24),
30+
err.kind(),
31+
);
32+
} else {
33+
content += &format!(" | {}\r\n", "^".repeat(24));
34+
}
35+
36+
for (idx, line) in post {
37+
content += &format!("{:>4} | {}\r\n", idx + 1, line);
38+
}
39+
40+
format!("{}\r\n{}\r\n{}", "-".repeat(79), content, "-".repeat(79))
41+
} else {
42+
"".to_string()
43+
};
44+
45+
if let Some(next_err) = err.source() {
46+
self.throw_error(format!(
47+
"{} caused by: {:#}\r\n{}",
48+
err, next_err, codeblock
49+
))
50+
} else {
51+
self.throw_error(format!("{}\r\n{}", err, codeblock))
52+
}
53+
}
54+
}
55+
56+
impl<'a> NeonMiniJinjaContext<'a> for FunctionContext<'a> {}
57+
58+
impl<'a> NeonMiniJinjaContext<'a> for TaskContext<'a> {}

‎packages/cubejs-backend-native/test/__snapshots__/jinja.test.ts.snap‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -211,7 +211,8 @@ exports[`Jinja (new api) render data-model.yml.jinja: data-model.yml.jinja 1`] =
211211
`;
212212

213213
exports[`Jinja (new api) render dump_context.yml.jinja: dump_context.yml.jinja 1`] = `
214-
"<pre></pre>
214+
"
215+
<pre></pre>
215216
216217
print:
217218
bool_true: true
@@ -239,7 +240,10 @@ exports[`Jinja (new api) render filters.yml.jinja: filters.yml.jinja 1`] = `
239240
exports[`Jinja (new api) render python.yml: python.yml 1`] = `
240241
"test:
241242
unsafe_string: \\"\\"\\\\\\"unsafe string\\\\\\" <>\\"\\"
242-
safe_string: \\"\\"safe string\\" <>\\""
243+
safe_string: \\"\\"safe string\\" <>\\"
244+
245+
dump:
246+
dict_as_obj: \\"{\\\\n \\\\\\"a_attr\\\\\\": String(\\\\n \\\\\\"value for attribute a\\\\\\",\\\\n Normal,\\\\n ),\\\\n}\\""
243247
`;
244248

245249
exports[`Jinja (new api) render template_error.jinja: template_error.jinja 1`] = `

‎packages/cubejs-backend-native/test/jinja.test.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ suite('Python model', () => {
8383
new_int_tuple: expect.any(Object),
8484
new_str_tuple: expect.any(Object),
8585
new_safe_string: expect.any(Object),
86+
new_object_from_dict: expect.any(Object),
8687
load_class_model: expect.any(Object),
8788
});
8889

@@ -115,6 +116,7 @@ darwinSuite('Scope Python model', () => {
115116
new_int_tuple: expect.any(Object),
116117
new_str_tuple: expect.any(Object),
117118
new_safe_string: expect.any(Object),
119+
new_object_from_dict: expect.any(Object),
118120
load_class_model: expect.any(Object),
119121
});
120122
});

‎packages/cubejs-backend-native/test/templates/dump_context.yml.jinja‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
{# TODO: We need stable sort for dump #}
12
<pre>{# debug() #}</pre>
23

34
print:

‎packages/cubejs-backend-native/test/templates/jinja-instance.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,16 @@ def new_str_tuple():
4949
def new_safe_string():
5050
return SafeString('"safe string" <>')
5151

52+
class MyCustomObject(dict):
53+
def __init__(self):
54+
self['a_attr'] = "value for attribute a"
55+
# TODO: We need stable sort for dump
56+
# self['b_attr'] = "value for attribute b"
57+
58+
@template.function
59+
def new_object_from_dict():
60+
return MyCustomObject()
61+
5262
@template.function
5363
def load_data_sync():
5464
client = MyApiClient("google.com")

‎packages/cubejs-backend-native/test/templates/python.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
{%- set unsafe_string = '"unsafe string" <>' -%}
22
{%- set safe_string = new_safe_string() -%}
3+
{%- set dict_as_obj = new_object_from_dict() -%}
34

45
test:
56
unsafe_string: "{{ unsafe_string }}"
67
safe_string: "{{ safe_string }}"
8+
9+
dump:
10+
dict_as_obj: {{ debug(dict_as_obj) }}

0 commit comments

Comments
 (0)