Skip to content

Commit e3e7e0e

Browse files
authored
feat(native): Support jinja inside yml in fallback build (without python) (#7203)
1 parent 9c3b859 commit e3e7e0e

File tree

21 files changed

+824
-727
lines changed

21 files changed

+824
-727
lines changed

.github/workflows/push.yml

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -432,7 +432,7 @@ jobs:
432432
integration-smoke:
433433
needs: [ latest-tag-sha ]
434434
runs-on: ubuntu-20.04
435-
timeout-minutes: 60
435+
timeout-minutes: 90
436436
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
437437

438438
strategy:
@@ -575,6 +575,22 @@ jobs:
575575
tag: tmp-dev
576576
fail-fast: false
577577
steps:
578+
- name: Maximize build space (disk space limitations)
579+
run: |
580+
echo "Before"
581+
df -h
582+
sudo apt-get remove -y 'php.*'
583+
sudo apt-get remove -y '^mongodb-.*'
584+
sudo apt-get remove -y '^mysql-.*'
585+
sudo apt-get autoremove -y
586+
sudo apt-get clean
587+
588+
sudo rm -rf /usr/share/dotnet
589+
sudo rm -rf /usr/local/lib/android
590+
sudo rm -rf /opt/ghc
591+
sudo rm -rf /opt/hostedtoolcache/CodeQL
592+
echo "After"
593+
df -h
578594
- name: Checkout
579595
uses: actions/checkout@v4
580596
- name: Set up QEMU

.github/workflows/rust-cubesql.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16
7272
shared-key: cubesql-${{ runner.OS }}-x86_64-unknown-linux-gnu-16
7373
- name: Install [email protected]
74-
uses: baptiste0928/cargo-install@v1
74+
uses: baptiste0928/cargo-install@v2
7575
with:
7676
crate: cargo-tarpaulin
7777
version: "0.20.1"

packages/cubejs-backend-native/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ once_cell = "1.10"
2727
libc = "0.2"
2828
findshlibs = "0.10.2"
2929
convert_case = "0.6.0"
30+
minijinja = { version = "1", features = ["json", "loader"] }
3031
# python
31-
minijinja = { version = "1", features = ["json", "loader"], optional = true }
3232
pyo3 = { version = "0.19", features = [], optional = true }
3333
pyo3-asyncio = { version = "0.19", features = ["tokio-runtime", "attributes"], optional = true }
3434

@@ -39,4 +39,4 @@ features = ["napi-1", "napi-4", "napi-6", "channel-api", "promise-api", "task-ap
3939

4040
[features]
4141
default = []
42-
python = ["pyo3", "pyo3-asyncio", "minijinja"]
42+
python = ["pyo3", "pyo3-asyncio"]

packages/cubejs-backend-native/js/index.ts

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -386,16 +386,15 @@ export class NativeInstance {
386386
}
387387

388388
public newJinjaEngine(options: { debugInfo?: boolean }): JinjaEngine {
389-
if (isFallbackBuild()) {
390-
throw new Error('Python (newJinjaEngine) is not supported in fallback build');
391-
}
392-
393389
return this.getNative().newJinjaEngine(options);
394390
}
395391

396392
public loadPythonContext(fileName: string, content: unknown): Promise<PythonCtx> {
397393
if (isFallbackBuild()) {
398-
throw new Error('Python (loadPythonContext) is not supported in fallback build');
394+
throw new Error(
395+
'Python (loadPythonContext) is not supported because you are using the fallback build of native extension. Read more: ' +
396+
'https://github.com/cube-js/cube/blob/master/packages/cubejs-backend-native/README.md#supported-architectures-and-platforms'
397+
);
399398
}
400399

401400
return this.getNative().pythonLoadModel(fileName, content);
Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
#[cfg(feature = "python")]
2+
use crate::cross::clrepr_python::PythonRef;
3+
#[cfg(feature = "python")]
4+
use crate::cross::py_in_js::{
5+
cl_repr_py_function_wrapper, BoxedJsPyFunctionWrapper, JsPyFunctionWrapper,
6+
};
7+
#[cfg(feature = "python")]
8+
use crate::utils::bind_method;
9+
use neon::prelude::*;
10+
use neon::result::Throw;
11+
use neon::types::JsDate;
12+
#[cfg(feature = "python")]
13+
use std::cell::RefCell;
14+
use std::collections::hash_map::{IntoIter, Iter, Keys};
15+
use std::collections::HashMap;
16+
use std::sync::Arc;
17+
18+
#[derive(Clone)]
19+
pub struct CLReprObject(pub(crate) HashMap<String, CLRepr>);
20+
21+
impl CLReprObject {
22+
pub fn new() -> Self {
23+
Self(HashMap::new())
24+
}
25+
26+
pub fn get(&self, key: &str) -> Option<&CLRepr> {
27+
self.0.get(key)
28+
}
29+
30+
pub fn insert(&mut self, key: String, value: CLRepr) -> Option<CLRepr> {
31+
self.0.insert(key, value)
32+
}
33+
34+
pub fn into_iter(self) -> IntoIter<String, CLRepr> {
35+
self.0.into_iter()
36+
}
37+
38+
pub fn iter(&self) -> Iter<String, CLRepr> {
39+
self.0.iter()
40+
}
41+
42+
pub fn keys(&self) -> Keys<'_, String, CLRepr> {
43+
self.0.keys()
44+
}
45+
}
46+
47+
impl std::fmt::Debug for CLReprObject {
48+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
49+
std::fmt::Debug::fmt(&self.0, f)
50+
}
51+
}
52+
53+
impl std::fmt::Display for CLReprObject {
54+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
55+
std::fmt::Debug::fmt(&self.0, f)
56+
}
57+
}
58+
59+
#[allow(unused)]
60+
#[derive(Debug)]
61+
pub enum CLReprKind {
62+
String,
63+
Bool,
64+
Float,
65+
Int,
66+
Tuple,
67+
Array,
68+
Object,
69+
JsFunction,
70+
#[cfg(feature = "python")]
71+
PythonRef,
72+
Null,
73+
}
74+
75+
/// Cross language representation is abstraction to transfer values between
76+
/// JavaScript and Python across Rust. Converting between two different languages requires
77+
/// to use Context which is available on the call (one for python and one for js), which result as
78+
/// blocking.
79+
#[derive(Debug, Clone)]
80+
pub enum CLRepr {
81+
String(String),
82+
Bool(bool),
83+
Float(f64),
84+
Int(i64),
85+
#[allow(dead_code)]
86+
Tuple(Vec<CLRepr>),
87+
Array(Vec<CLRepr>),
88+
Object(CLReprObject),
89+
JsFunction(Arc<Root<JsFunction>>),
90+
#[cfg(feature = "python")]
91+
PythonRef(PythonRef),
92+
Null,
93+
}
94+
95+
impl std::fmt::Display for CLRepr {
96+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97+
std::fmt::Display::fmt(&self, f)
98+
}
99+
}
100+
101+
#[cfg(feature = "python")]
102+
struct IntoJsContext {
103+
parent_key_name: Option<String>,
104+
}
105+
106+
impl CLRepr {
107+
pub fn is_null(&self) -> bool {
108+
matches!(self, CLRepr::Null)
109+
}
110+
111+
pub fn downcast_to_object(self) -> CLReprObject {
112+
match self {
113+
CLRepr::Object(obj) => obj,
114+
_ => panic!("downcast_to_object rejected, actual: {:?}", self.kind()),
115+
}
116+
}
117+
118+
#[allow(unused)]
119+
pub fn kind(&self) -> CLReprKind {
120+
match self {
121+
CLRepr::String(_) => CLReprKind::String,
122+
CLRepr::Bool(_) => CLReprKind::Bool,
123+
CLRepr::Float(_) => CLReprKind::Float,
124+
CLRepr::Int(_) => CLReprKind::Int,
125+
CLRepr::Tuple(_) => CLReprKind::Tuple,
126+
CLRepr::Array(_) => CLReprKind::Array,
127+
CLRepr::Object(_) => CLReprKind::Object,
128+
CLRepr::JsFunction(_) => CLReprKind::JsFunction,
129+
#[cfg(feature = "python")]
130+
CLRepr::PythonRef(_) => CLReprKind::PythonRef,
131+
CLRepr::Null => CLReprKind::Null,
132+
}
133+
}
134+
135+
/// Convert javascript value to CLRepr
136+
pub fn from_js_ref<'a, C: Context<'a>>(
137+
from: Handle<'a, JsValue>,
138+
cx: &mut C,
139+
) -> Result<Self, Throw> {
140+
if from.is_a::<JsString, _>(cx) {
141+
let v = from.downcast_or_throw::<JsString, _>(cx)?;
142+
Ok(CLRepr::String(v.value(cx)))
143+
} else if from.is_a::<JsArray, _>(cx) {
144+
let v = from.downcast_or_throw::<JsArray, _>(cx)?;
145+
let el = v.to_vec(cx)?;
146+
147+
let mut r = Vec::with_capacity(el.len());
148+
149+
for el in el {
150+
r.push(Self::from_js_ref(el, cx)?)
151+
}
152+
153+
Ok(CLRepr::Array(r))
154+
} else if from.is_a::<JsObject, _>(cx) {
155+
let mut obj = CLReprObject::new();
156+
157+
let v = from.downcast_or_throw::<JsObject, _>(cx)?;
158+
let properties = v.get_own_property_names(cx)?;
159+
for i in 0..properties.len(cx) {
160+
let property: Handle<JsString> = properties.get(cx, i)?;
161+
let property_val = v.get_value(cx, property)?;
162+
163+
obj.insert(property.value(cx), Self::from_js_ref(property_val, cx)?);
164+
}
165+
166+
Ok(CLRepr::Object(obj))
167+
} else if from.is_a::<JsBoolean, _>(cx) {
168+
let v = from.downcast_or_throw::<JsBoolean, _>(cx)?;
169+
Ok(CLRepr::Bool(v.value(cx)))
170+
} else if from.is_a::<JsNumber, _>(cx) {
171+
let v = from.downcast_or_throw::<JsNumber, _>(cx)?.value(cx);
172+
173+
if v == (v as i64) as f64 {
174+
Ok(CLRepr::Int(v as i64))
175+
} else {
176+
Ok(CLRepr::Float(v))
177+
}
178+
} else if from.is_a::<JsNull, _>(cx) || from.is_a::<JsUndefined, _>(cx) {
179+
Ok(CLRepr::Null)
180+
} else if from.is_a::<JsPromise, _>(cx) {
181+
cx.throw_error("Unsupported conversion from JsPromise to CLRepr")?
182+
} else if from.is_a::<JsDate, _>(cx) {
183+
cx.throw_error("Unsupported conversion from JsDate to CLRepr")?
184+
} else if from.is_a::<JsFunction, _>(cx) {
185+
let fun = from.downcast_or_throw::<JsFunction, _>(cx)?;
186+
let fun_root = fun.root(cx);
187+
188+
Ok(CLRepr::JsFunction(Arc::new(fun_root)))
189+
} else {
190+
#[cfg(feature = "python")]
191+
if from.is_a::<BoxedJsPyFunctionWrapper, _>(cx) {
192+
let ref_wrap = from.downcast_or_throw::<BoxedJsPyFunctionWrapper, _>(cx)?;
193+
let fun = ref_wrap.borrow().get_fun().clone();
194+
195+
return Ok(CLRepr::PythonRef(PythonRef::PyFunction(fun)));
196+
}
197+
198+
cx.throw_error(format!("Unsupported conversion from {:?} to CLRepr", from))
199+
}
200+
}
201+
202+
fn into_js_impl<'a, C: Context<'a>>(
203+
from: CLRepr,
204+
cx: &mut C,
205+
#[cfg(feature = "python")] tcx: IntoJsContext,
206+
) -> JsResult<'a, JsValue> {
207+
Ok(match from {
208+
CLRepr::String(v) => cx.string(v).upcast(),
209+
CLRepr::Bool(v) => cx.boolean(v).upcast(),
210+
CLRepr::Float(v) => cx.number(v).upcast(),
211+
CLRepr::Int(v) => cx.number(v as f64).upcast(),
212+
CLRepr::Tuple(arr) | CLRepr::Array(arr) => {
213+
let r = cx.empty_array();
214+
215+
for (k, v) in arr.into_iter().enumerate() {
216+
let vv = Self::into_js_impl(
217+
v,
218+
cx,
219+
#[cfg(feature = "python")]
220+
IntoJsContext {
221+
parent_key_name: None,
222+
},
223+
)?;
224+
r.set(cx, k as u32, vv)?;
225+
}
226+
227+
r.upcast()
228+
}
229+
CLRepr::Object(obj) => {
230+
let r = cx.empty_object();
231+
232+
for (k, v) in obj.into_iter() {
233+
let r_k = cx.string(k.clone());
234+
let r_v = Self::into_js_impl(
235+
v,
236+
cx,
237+
#[cfg(feature = "python")]
238+
IntoJsContext {
239+
parent_key_name: Some(k),
240+
},
241+
)?;
242+
243+
r.set(cx, r_k, r_v)?;
244+
}
245+
246+
r.upcast()
247+
}
248+
#[cfg(feature = "python")]
249+
CLRepr::PythonRef(py_ref) => match py_ref {
250+
PythonRef::PyFunction(py_fn) => {
251+
let wrapper = JsPyFunctionWrapper::new(py_fn, tcx.parent_key_name);
252+
let obj_this = cx.boxed(RefCell::new(wrapper)).upcast::<JsValue>();
253+
254+
let cl_repr_fn = JsFunction::new(cx, cl_repr_py_function_wrapper)?;
255+
let binded_fun = bind_method(cx, cl_repr_fn, obj_this)?;
256+
257+
binded_fun.upcast()
258+
}
259+
PythonRef::PyExternalFunction(py_fn) => {
260+
let wrapper = JsPyFunctionWrapper::new(py_fn, tcx.parent_key_name);
261+
let external_obj = cx.boxed(RefCell::new(wrapper)).upcast::<JsValue>();
262+
263+
external_obj.upcast()
264+
}
265+
PythonRef::PyObject(_) => {
266+
return cx.throw_error("Unable to represent PyObject in JS")
267+
}
268+
},
269+
CLRepr::Null => cx.undefined().upcast(),
270+
CLRepr::JsFunction(fun) => {
271+
let unwrapper_fun =
272+
Arc::try_unwrap(fun).expect("Unable to unwrap Arc on Root<JsFunction>");
273+
274+
unwrapper_fun.into_inner(cx).upcast()
275+
}
276+
})
277+
}
278+
279+
pub fn into_js<'a, C: Context<'a>>(self, cx: &mut C) -> JsResult<'a, JsValue> {
280+
Self::into_js_impl(
281+
self,
282+
cx,
283+
#[cfg(feature = "python")]
284+
IntoJsContext {
285+
parent_key_name: None,
286+
},
287+
)
288+
}
289+
}

0 commit comments

Comments
 (0)