Skip to content

Commit fb59e59

Browse files
ovrovr
authored
feat(schema): Support multitenancy for jinja templates (#6793)
1 parent f1389be commit fb59e59

File tree

5 files changed

+138
-143
lines changed

5 files changed

+138
-143
lines changed

‎packages/cubejs-backend-native/js/index.ts‎

Lines changed: 7 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -277,21 +277,12 @@ export const pythonLoadConfig = async (context: string, options: { file: string
277277
return config;
278278
}
279279

280-
export const initJinjaEngine = (options: { debugInfo?: boolean }): void => {
281-
const native = loadNative();
282-
return native.initJinjaEngine(options);
283-
};
284-
export const loadTemplate = (templateName: string, templateContent: string): void => {
285-
const native = loadNative();
286-
return native.loadTemplate(templateName, templateContent);
287-
};
288-
289-
export const clearTemplates = (): void => {
290-
const native = loadNative();
291-
return native.clearTemplates();
292-
};
280+
export interface JinjaEngine {
281+
loadTemplate(templateName: string, templateContent: string): void;
282+
renderTemplate(templateName: string, context: unknown): string;
283+
}
293284

294-
export const renderTemplate = (templateName: string, context: unknown): string => {
295-
const native = loadNative();
296-
return native.renderTemplate(templateName, context);
285+
export const newJinjaEngine = (options: { debugInfo?: boolean }): JinjaEngine => {
286+
const native = loadNative();
287+
return native.newJinjaEngine(options);
297288
};

‎packages/cubejs-backend-native/src/python/template/entry.rs‎

Lines changed: 95 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,12 @@
11
use crate::python::cross::{CLRepr, CLReprObject};
22
use crate::python::template::mj_value::to_minijinja_value;
3-
use log::{error, trace};
3+
use crate::utils::bind_method;
4+
use log::trace;
45
use minijinja as mj;
56
use neon::context::Context;
67
use neon::prelude::*;
7-
use once_cell::sync::OnceCell;
8+
use std::cell::RefCell;
89
use std::error::Error;
9-
use std::sync::Mutex;
1010

1111
trait NeonMiniJinjaContext {
1212
fn throw_from_mj_error<T>(&mut self, err: mj::Error) -> NeonResult<T>;
@@ -60,130 +60,128 @@ impl<'a> NeonMiniJinjaContext for FunctionContext<'a> {
6060
}
6161
}
6262

63-
#[derive(Debug)]
64-
struct EngineOptions {
65-
debug_info: bool,
63+
struct JinjaEngine {
64+
inner: mj::Environment<'static>,
6665
}
6766

68-
static TEMPLATE_ENGINE: OnceCell<Mutex<mj::Environment>> = OnceCell::new();
69-
70-
fn init_template_engine<'a, C: Context<'a>>(_cx: &mut C, opts: EngineOptions) -> NeonResult<()> {
71-
let mut engine = mj::Environment::new();
72-
engine.set_debug(opts.debug_info);
73-
engine.add_function(
74-
"env_var",
75-
|var_name: String, var_default: Option<String>, _state: &minijinja::State| {
76-
if let Ok(value) = std::env::var(&var_name) {
77-
return Ok(mj::value::Value::from(value));
78-
}
79-
80-
if let Some(var_default) = var_default {
81-
return Ok(mj::value::Value::from(var_default));
82-
}
83-
84-
let err = minijinja::Error::new(
85-
mj::ErrorKind::InvalidOperation,
86-
format!("unknown env variable {}", var_name),
87-
);
67+
impl Finalize for JinjaEngine {}
68+
69+
impl JinjaEngine {
70+
fn new(cx: &mut FunctionContext) -> NeonResult<Self> {
71+
let options = cx.argument::<JsObject>(0)?;
72+
73+
let debug_info = options
74+
.get_value(cx, "debugInfo")?
75+
.downcast_or_throw::<JsBoolean, _>(cx)?
76+
.value(cx);
77+
78+
let mut engine = mj::Environment::new();
79+
engine.set_debug(debug_info);
80+
engine.add_function(
81+
"env_var",
82+
|var_name: String, var_default: Option<String>, _state: &minijinja::State| {
83+
if let Ok(value) = std::env::var(&var_name) {
84+
return Ok(mj::value::Value::from(value));
85+
}
86+
87+
if let Some(var_default) = var_default {
88+
return Ok(mj::value::Value::from(var_default));
89+
}
90+
91+
let err = minijinja::Error::new(
92+
mj::ErrorKind::InvalidOperation,
93+
format!("unknown env variable {}", var_name),
94+
);
8895

89-
Err(err)
90-
},
91-
);
96+
Err(err)
97+
},
98+
);
9299

93-
if let Err(_) = TEMPLATE_ENGINE.set(Mutex::new(engine)) {
94-
// I am working on a new jinja engine implementation on top of isolated instances per tenant
95-
// to support multi tenancy
96-
#[cfg(debug_assertions)]
97-
error!("Unable to init jinja engine, it was already started");
100+
Ok(Self { inner: engine })
98101
}
99-
100-
Ok(())
101102
}
102103

103-
fn template_engine<'a, C: Context<'a>>(
104-
cx: &mut C,
105-
) -> NeonResult<&'static Mutex<mj::Environment<'static>>> {
106-
if let Some(engine) = TEMPLATE_ENGINE.get() {
107-
Ok(engine)
108-
} else {
109-
cx.throw_error("Unable to get jinja engine: It was not initialized".to_string())
110-
}
111-
}
104+
type BoxedJinjaEngine = JsBox<RefCell<JinjaEngine>>;
112105

113-
fn init_jinja_engine(mut cx: FunctionContext) -> JsResult<JsUndefined> {
114-
let options = cx.argument::<JsObject>(0)?;
106+
impl JinjaEngine {
107+
fn render_template(mut cx: FunctionContext) -> JsResult<JsString> {
108+
#[cfg(build = "debug")]
109+
trace!("JinjaEngine.render_template");
115110

116-
let debug_info: Handle<JsBoolean> = options
117-
.get_value(&mut cx, "debugInfo")?
118-
.downcast_or_throw(&mut cx)?;
111+
let this = cx
112+
.this()
113+
.downcast_or_throw::<BoxedJinjaEngine, _>(&mut cx)?;
119114

120-
let options = EngineOptions {
121-
debug_info: debug_info.value(&mut cx),
122-
};
123-
init_template_engine(&mut cx, options)?;
115+
let template_name = cx.argument::<JsString>(0)?;
116+
let template_ctx = CLRepr::from_js_ref(cx.argument::<JsValue>(1)?, &mut cx)?;
124117

125-
Ok(cx.undefined())
126-
}
118+
let engine = &this.borrow().inner;
119+
let template = match engine.get_template(&template_name.value(&mut cx)) {
120+
Ok(t) => t,
121+
Err(err) => {
122+
trace!("jinja get template error: {:?}", err);
127123

128-
fn load_template(mut cx: FunctionContext) -> JsResult<JsUndefined> {
129-
let template_name = cx.argument::<JsString>(0)?;
130-
let template_content = cx.argument::<JsString>(1)?;
124+
return cx.throw_from_mj_error(err);
125+
}
126+
};
131127

132-
let mut engine = template_engine(&mut cx)?.lock().unwrap();
128+
let mut ctx = CLReprObject::new();
129+
ctx.insert("COMPILE_CONTEXT".to_string(), template_ctx);
133130

134-
if let Err(err) = engine.add_template_owned(
135-
template_name.value(&mut cx),
136-
template_content.value(&mut cx),
137-
) {
138-
trace!("jinja load error: {:?}", err);
131+
let compile_context = to_minijinja_value(CLRepr::Object(ctx));
132+
match template.render(compile_context) {
133+
Ok(r) => Ok(cx.string(r)),
134+
Err(err) => {
135+
trace!("jinja render template error: {:?}", err);
139136

140-
return cx.throw_from_mj_error(err);
137+
cx.throw_from_mj_error(err)
138+
}
139+
}
141140
}
142141

143-
Ok(cx.undefined())
144-
}
145-
146-
fn clear_templates(mut cx: FunctionContext) -> JsResult<JsUndefined> {
147-
let mut engine = template_engine(&mut cx)?.lock().unwrap();
148-
engine.clear_templates();
149-
150-
Ok(cx.undefined())
151-
}
142+
fn load_template(mut cx: FunctionContext) -> JsResult<JsUndefined> {
143+
#[cfg(build = "debug")]
144+
trace!("JinjaEngine.load_template");
152145

153-
fn render_template(mut cx: FunctionContext) -> JsResult<JsString> {
154-
let template_name = cx.argument::<JsString>(0)?;
155-
let template_ctx = CLRepr::from_js_ref(cx.argument::<JsValue>(1)?, &mut cx)?;
146+
let this = cx
147+
.this()
148+
.downcast_or_throw::<BoxedJinjaEngine, _>(&mut cx)?;
156149

157-
let engine = template_engine(&mut cx)?.lock().unwrap();
150+
let template_name = cx.argument::<JsString>(0)?;
151+
let template_content = cx.argument::<JsString>(1)?;
158152

159-
let template = match engine.get_template(&template_name.value(&mut cx)) {
160-
Ok(t) => t,
161-
Err(err) => {
162-
trace!("jinja get template error: {:?}", err);
153+
if let Err(err) = this.borrow_mut().inner.add_template_owned(
154+
template_name.value(&mut cx),
155+
template_content.value(&mut cx),
156+
) {
157+
trace!("jinja load error: {:?}", err);
163158

164159
return cx.throw_from_mj_error(err);
165160
}
166-
};
167161

168-
let mut ctx = CLReprObject::new();
169-
ctx.insert("COMPILE_CONTEXT".to_string(), template_ctx);
162+
Ok(cx.undefined())
163+
}
170164

171-
let compile_context = to_minijinja_value(CLRepr::Object(ctx));
172-
match template.render(compile_context) {
173-
Ok(r) => Ok(cx.string(r)),
174-
Err(err) => {
175-
trace!("jinja render template error: {:?}", err);
165+
fn js_new(mut cx: FunctionContext) -> JsResult<JsObject> {
166+
let engine = Self::new(&mut cx).or_else(|err| cx.throw_error(err.to_string()))?;
176167

177-
cx.throw_from_mj_error(err)
178-
}
168+
let obj = cx.empty_object();
169+
let obj_this = cx.boxed(RefCell::new(engine)).upcast::<JsValue>();
170+
171+
let render_template_fn = JsFunction::new(&mut cx, JinjaEngine::render_template)?;
172+
let render_template_fn = bind_method(&mut cx, render_template_fn, obj_this)?;
173+
obj.set(&mut cx, "renderTemplate", render_template_fn)?;
174+
175+
let load_template_fn = JsFunction::new(&mut cx, JinjaEngine::load_template)?;
176+
let load_template_fn = bind_method(&mut cx, load_template_fn, obj_this)?;
177+
obj.set(&mut cx, "loadTemplate", load_template_fn)?;
178+
179+
Ok(obj)
179180
}
180181
}
181182

182183
pub fn template_register_module(cx: &mut ModuleContext) -> NeonResult<()> {
183-
cx.export_function("initJinjaEngine", init_jinja_engine)?;
184-
cx.export_function("loadTemplate", load_template)?;
185-
cx.export_function("clearTemplates", clear_templates)?;
186-
cx.export_function("renderTemplate", render_template)?;
184+
cx.export_function("newJinjaEngine", JinjaEngine::js_new)?;
187185

188186
Ok(())
189187
}

‎packages/cubejs-backend-native/test/jinja.test.ts‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,27 +2,28 @@ import fs from 'fs';
22
import path from 'path';
33

44
import * as native from '../js';
5+
import { JinjaEngine, newJinjaEngine } from '../js';
56

67
const suite = native.isFallbackBuild() ? xdescribe : describe;
78

8-
function loadTemplateFile(fileName: string): void {
9+
function loadTemplateFile(engine: native.JinjaEngine, fileName: string): void {
910
const content = fs.readFileSync(path.join(process.cwd(), 'test', 'templates', fileName), 'utf8');
1011

11-
native.loadTemplate(fileName, content);
12+
engine.loadTemplate(fileName, content);
1213
}
1314

14-
function testTemplateBySnapshot(templateName: string, ctx: unknown) {
15+
function testTemplateBySnapshot(engine: JinjaEngine, templateName: string, ctx: unknown) {
1516
test(`render ${templateName}`, async () => {
16-
const actual = native.renderTemplate(templateName, ctx);
17+
const actual = engine.renderTemplate(templateName, ctx);
1718

1819
expect(actual).toMatchSnapshot(templateName);
1920
});
2021
}
2122

22-
function testLoadBrokenTemplateBySnapshot(templateName: string) {
23+
function testLoadBrokenTemplateBySnapshot(engine: JinjaEngine, templateName: string) {
2324
test(`render ${templateName}`, async () => {
2425
try {
25-
loadTemplateFile(templateName);
26+
loadTemplateFile(engine, templateName);
2627

2728
throw new Error(`Template ${templateName} should throw an error!`);
2829
} catch (e) {
@@ -32,21 +33,20 @@ function testLoadBrokenTemplateBySnapshot(templateName: string) {
3233
}
3334

3435
suite('Jinja', () => {
35-
beforeAll(async () => {
36-
native.initJinjaEngine({
37-
debugInfo: true
38-
});
39-
native.clearTemplates();
36+
const jinjaEngine = native.newJinjaEngine({
37+
debugInfo: true
38+
});
4039

41-
loadTemplateFile('.utils.jinja');
42-
loadTemplateFile('dump_context.yml.jinja');
40+
beforeAll(async () => {
41+
loadTemplateFile(jinjaEngine, '.utils.jinja');
42+
loadTemplateFile(jinjaEngine, 'dump_context.yml.jinja');
4343

4444
for (let i = 1; i < 9; i++) {
45-
loadTemplateFile(`0${i}.yml.jinja`);
45+
loadTemplateFile(jinjaEngine, `0${i}.yml.jinja`);
4646
}
4747
});
4848

49-
testTemplateBySnapshot('dump_context.yml.jinja', {
49+
testTemplateBySnapshot(jinjaEngine, 'dump_context.yml.jinja', {
5050
bool_true: true,
5151
bool_false: false,
5252
string: 'test string',
@@ -60,9 +60,9 @@ suite('Jinja', () => {
6060
userId: 1,
6161
}
6262
});
63-
testLoadBrokenTemplateBySnapshot('template_error.jinja');
63+
testLoadBrokenTemplateBySnapshot(jinjaEngine, 'template_error.jinja');
6464

6565
for (let i = 1; i < 9; i++) {
66-
testTemplateBySnapshot(`0${i}.yml.jinja`, {});
66+
testTemplateBySnapshot(jinjaEngine, `0${i}.yml.jinja`, {});
6767
}
6868
});

‎packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js‎

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { parse } from '@babel/parser';
66
import babelGenerator from '@babel/generator';
77
import babelTraverse from '@babel/traverse';
88
import R from 'ramda';
9-
import { loadTemplate, clearTemplates, isFallbackBuild, initJinjaEngine } from '@cubejs-backend/native';
9+
import { clearTemplates, isFallbackBuild, initJinjaEngine } from '@cubejs-backend/native';
1010

1111
import { getEnv, isNativeSupported } from '@cubejs-backend/shared';
1212
import { AbstractExtension } from '../extensions';
@@ -57,15 +57,6 @@ export class DataSchemaCompiler {
5757
*/
5858
async doCompile() {
5959
const files = await this.repository.dataSchemaFiles();
60-
const hasJinjaTemplate = files.find((file) => file.fileName.endsWith('.jinja'));
61-
62-
if (hasJinjaTemplate && NATIVE_IS_SUPPORTED && !isFallbackBuild()) {
63-
initJinjaEngine({
64-
debugInfo: getEnv('devMode'),
65-
});
66-
clearTemplates();
67-
}
68-
6960
const toCompile = files.filter((f) => !this.filesToCompile || this.filesToCompile.indexOf(f.fileName) !== -1);
7061

7162
const errorsReport = new ErrorReporter(null, [], this.errorReport);
@@ -113,7 +104,7 @@ export class DataSchemaCompiler {
113104
);
114105
}
115106

116-
loadTemplate(file.fileName, file.content);
107+
this.yamlCompiler.getJinjaEngine().loadTemplate(file.fileName, file.content);
117108

118109
return file;
119110
} else if (R.endsWith('.yml', file.fileName) || R.endsWith('.yaml', file.fileName)) {

0 commit comments

Comments
 (0)