feat(backend-native): Allow importing modules from python files within cube.py and globals.py (#9490)

* feat(backend-native): Allow importing modules from python files within cube.py

* update tests

* Allow importing modules from python files within globals.py

* tests

* another test

* Add docs

* more tests

* extract sys path to fn

---------

Co-authored-by: Igor Lukanin <[email protected]>

Author: KSDaemon

docs/pages/product/configuration.mdx

@@ -1,8 +1,3 @@
----
-redirect_from:
-  - /configuration/overview
----
-
 # Overview
 
 Cube is configured via [environment variables][link-env-vars] and
@@ -103,6 +98,9 @@ module.exports = {
 
 Both ways are equivalent; when in doubt, use Python.
 
+You can read more about [Python][ref-python] and [JavaScript][ref-javascript] support
+in the dynamic data modeling section of the documentation.
+
 ### Cube Core
 
 When using Docker, ensure that the configuration file and your [data model
@@ -112,7 +110,8 @@ Docker container.
 ### Cube Cloud
 
 You can edit the configuration file by going into <Btn>Development Mode</Btn>
-and navigating to the <Btn>Data Model</Btn> page.
+and navigating to <Btn>[Data Model][ref-data-model]</Btn> or <Btn>[Visual
+Model][ref-visual-model]</Btn> pages.
 
 ## Runtimes and dependencies
 
@@ -178,4 +177,8 @@ mode does the following:
 [link-docker-env-vars]: https://docs.docker.com/compose/environment-variables/set-environment-variables/
 [ref-mls]: /product/auth/member-level-security
 [link-current-python-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L13
-[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1
+[link-current-nodejs-version]: https://github.com/cube-js/cube/blob/master/packages/cubejs-docker/latest.Dockerfile#L1
+[ref-data-model]: /product/workspace/data-model
+[ref-visual-model]: /product/workspace/visual-model
+[ref-python]: /product/data-modeling/dynamic/jinja#python
+[ref-javascript]: /product/data-modeling/dynamic/javascript

docs/pages/product/data-modeling/dynamic/jinja.mdx

@@ -155,13 +155,20 @@ class SafeString(str):
 
 ## Python
 
-You can declare and invoke Python functions from within a Jinja template. This
-allows the reuse of existing code to generate data models. Cube uses Python 3.9 to execute Python code. 
-It also installs packages listed in the `requirements.txt` with pip on the startup.
+### Template context
 
-These helper functions must be located in `model/globals.py` file or explicitly loaded from the YAML files.
-In the following example, we declare a function called `load_data()` which will load data from a remote
-API endpoint. We will then use the function to generate a data model in Cube.
+You can use Python to declare functions that can be invoked and variables that can be
+referenced from within a Jinja template. These functions and variables must be defined
+in `model/globals.py` file and registered in the `TemplateContext` instance.
+
+<ReferenceBox>
+
+See the [`TemplateContext` reference][ref-cube-template-context] for more details.
+
+</ReferenceBox>
+
+In the following example, we declare a function called `load_data` that supposedly loads
+data from a remote API endpoint. We will then use the function to generate a data model:
 
 ```python
 from cube import TemplateContext
@@ -237,12 +244,46 @@ cubes:
   {%- endfor %}
 ```
 
-<ReferenceBox>
+### Imports
 
-If you'd like to split your Python code into several files, see
-[this issue](https://github.com/cube-js/cube/issues/8443#issuecomment-2219804266).
+In the `model/globals.py` file (or the `cube.py` configuration file), you can
+import modules from the current directory. In the following example, we import a function
+from the `utils` module and use it to populate a variable in the template context:
 
-</ReferenceBox>
+```python filename="model/utils.py"
+def answer_to_main_question() -> str:
+  return "42"
+```
+
+```python filename="model/globals.py"
+from cube import TemplateContext
+from utils import answer_to_main_question
+
+template = TemplateContext()
+
+answer = answer_to_main_question()
+template.add_variable('answer', answer)
+```
+### Dependencies
+
+If you need to use dependencies in your dynamic data model (or your `cube.py`
+configuration file), you can list them in the `requirements.txt` file in the root
+directory of your Cube deployment. They will be automatically installed with `pip` on
+the startup.
+
+<InfoBox>
+
+[`cube` package][ref-cube-package] is available out of the box, it doesn't need to be
+listed in `requirements.txt`.
+
+</InfoBox>
+
+If you use dbt for data transformation, you might find the [`cube_dbt`
+package][ref-cube-dbt-package] useful. It provides a set of utilities that simplify
+defining the data model in YAML [based on dbt models][ref-cube-with-dbt].
+
+If you need to use dependencies with native extensions, build a [custom Docker
+image][ref-docker-image-extension].
 
 
 [jinja]: https://jinja.palletsprojects.com/
@@ -253,4 +294,9 @@ If you'd like to split your Python code into several files, see
 [jinja-docs-autoescaping]: https://jinja.palletsprojects.com/en/3.1.x/api/#autoescaping
 [jinja-docs-filters-safe]: https://jinja.palletsprojects.com/en/3.1.x/templates/#jinja-filters.safe
 [ref-cube-dbt]: /reference/python/cube_dbt
-[ref-visual-model]: /product/workspace/visual-model
+[ref-visual-model]: /product/workspace/visual-model
+[ref-docker-image-extension]: /product/deployment/core#extend-the-docker-image
+[ref-cube-package]: /reference/python/cube
+[ref-cube-template-context]: /reference/python/cube#templatecontext-class
+[ref-cube-dbt-package]: /reference/python/cube_dbt
+[ref-cube-with-dbt]: /guides/dbt

packages/cubejs-backend-native/src/python/entry.rs

@@ -6,6 +6,19 @@ use crate::python::runtime::py_runtime_init;
 use neon::prelude::*;
 use pyo3::prelude::*;
 use pyo3::types::{PyDict, PyFunction, PyList, PyString, PyTuple};
+use std::path::Path;
+
+fn extend_sys_path(py: Python, file_name: &String) -> PyResult<()> {
+    let sys_path = py.import("sys")?.getattr("path")?.downcast::<PyList>()?;
+
+    let config_dir = Path::new(&file_name)
+        .parent()
+        .unwrap_or_else(|| Path::new("."));
+    let config_dir_str = config_dir.to_str().unwrap_or(".");
+
+    sys_path.insert(0, PyString::new(py, config_dir_str))?;
+    Ok(())
+}
 
 fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
     let file_content_arg = cx.argument::<JsString>(0)?.value(&mut cx);
@@ -20,6 +33,8 @@ fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
     py_runtime_init(&mut cx, channel.clone())?;
 
     let conf_res = Python::with_gil(|py| -> PyResult<CubeConfigPy> {
+        extend_sys_path(py, &options_file_name)?;
+
         let cube_code = include_str!(concat!(
             env!("CARGO_MANIFEST_DIR"),
             "/python/cube/src/__init__.py"
@@ -61,6 +76,8 @@ fn python_load_model(mut cx: FunctionContext) -> JsResult<JsPromise> {
     py_runtime_init(&mut cx, channel.clone())?;
 
     let conf_res = Python::with_gil(|py| -> PyResult<CubePythonModel> {
+        extend_sys_path(py, &model_file_name)?;
+
         let cube_code = include_str!(concat!(
             env!("CARGO_MANIFEST_DIR"),
             "/python/cube/src/__init__.py"

packages/cubejs-backend-native/test/config.py

@@ -1,4 +1,5 @@
 from cube import config, file_repository
+from utils import test_function
 
 config.schema_path = "models"
 config.pg_sql_port = 5555
@@ -7,6 +8,7 @@
 
 @config
 def query_rewrite(query, ctx):
+    query = test_function(query)
     print("[python] query_rewrite query=", query, " ctx=", ctx)
     return query
 

packages/cubejs-backend-native/test/globals.py

@@ -0,0 +1,16 @@
+from cube import TemplateContext
+import os
+from utils import answer_to_main_question
+from subdir_for_test.meta import main_question
+
+
+template = TemplateContext()
+
+value_or_none = os.getenv('MY_ENV_VAR')
+template.add_variable('value_or_none', value_or_none)
+
+value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value')
+template.add_variable('value_or_default', value_or_default)
+
+template.add_variable('main_question', main_question())
+template.add_variable('answer_to_main_question', answer_to_main_question())

packages/cubejs-backend-native/test/globals_w_import_path.py

@@ -0,0 +1,17 @@
+from cube import TemplateContext
+import sys
+import os
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from utils import answer_to_main_question
+
+template = TemplateContext()
+
+value_or_none = os.getenv('MY_ENV_VAR')
+template.add_variable('value_or_none', value_or_none)
+
+value_or_default = os.getenv('MY_OTHER_ENV_VAR', 'my_default_value')
+template.add_variable('value_or_default', value_or_default)
+
+template.add_variable('answer_to_main_question', answer_to_main_question())

packages/cubejs-backend-native/test/python.test.ts

@@ -9,16 +9,17 @@ const suite = native.isFallbackBuild() ? xdescribe : describe;
 const darwinSuite = process.platform === 'darwin' && !native.isFallbackBuild() ? describe : xdescribe;
 
 async function loadConfigurationFile(fileName: string) {
-  const content = await fs.readFile(path.join(process.cwd(), 'test', fileName), 'utf8');
+  const fullFileName = path.join(process.cwd(), 'test', fileName);
+  const content = await fs.readFile(fullFileName, 'utf8');
   console.log('content', {
     content,
-    fileName
+    fileName: fullFileName
   });
 
   const config = await native.pythonLoadConfig(
     content,
     {
-      fileName
+      fileName: fullFileName
     }
   );
 
@@ -27,6 +28,26 @@ async function loadConfigurationFile(fileName: string) {
   return config;
 }
 
+const nativeInstance = new native.NativeInstance();
+
+suite('Python Models', () => {
+  test('models import', async () => {
+    const fullFileName = path.join(process.cwd(), 'test', 'globals.py');
+    const content = await fs.readFile(fullFileName, 'utf8');
+
+    // Just checking it won't fail
+    await nativeInstance.loadPythonContext(fullFileName, content);
+  });
+
+  test('models import with sys.path changed', async () => {
+    const fullFileName = path.join(process.cwd(), 'test', 'globals_w_import_path.py');
+    const content = await fs.readFile(fullFileName, 'utf8');
+
+    // Just checking it won't fail
+    await nativeInstance.loadPythonContext(fullFileName, content);
+  });
+});
+
 suite('Python Config', () => {
   let config: PyConfiguration;
 

packages/cubejs-backend-native/test/subdir_for_test/meta.py

@@ -0,0 +1,8 @@
+# Separate file module for testing python imports
+
+# Simple test function
+def test_meta_function(query: dict) -> dict:
+  return query
+
+def main_question() -> str:
+  return "Why?"

packages/cubejs-backend-native/test/utils.py

@@ -0,0 +1,8 @@
+# Separate file module for testing python imports
+
+# Simple test function
+def test_function(query: dict) -> dict:
+  return query
+
+def answer_to_main_question() -> str:
+  return "42"