chore(native): Make python as optional feature

Author: ovr

.github/workflows/rust-cubesql.yml

@@ -120,13 +120,14 @@ jobs:
         run: cd rust/cubesql && cargo test
 
   native_linux:
-    needs: [lint,unit,unit_legacy]
+    needs: [lint]
     runs-on: ubuntu-20.04
     timeout-minutes: 60
-    name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }}
+    name: Build Linux GNU ${{ matrix.node-version }}.x ${{ matrix.target }} with Python ${{ matrix.python-version }}
     strategy:
       matrix:
         node-version: [14, 16, 18]
+        python-version: [3.11, "false"]
         target: ["x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]
       fail-fast: false
     container:
@@ -177,10 +178,16 @@ jobs:
           command: yarn install --frozen-lockfile
       - name: Lerna tsc
         run: yarn tsc
-      - name: Build native
+      - name: Build native (fallback)
+        if: (matrix.python-version == 'false')
         env:
           CARGO_BUILD_TARGET: ${{ matrix.target }}
-        run: cd packages/cubejs-backend-native && yarn run native:build
+        run: cd packages/cubejs-backend-native && yarn run native:build-debug
+      - name: Build native (with Python)
+        if: (matrix.python-version != 'false')
+        env:
+          CARGO_BUILD_TARGET: ${{ matrix.target }}
+        run: cd packages/cubejs-backend-native && yarn run native:build-debug-python
       - name: Test native (GNU only)
         if: (matrix.target == 'x86_64-unknown-linux-gnu')
         env:
@@ -194,16 +201,17 @@ jobs:
         run: cd packages/cubejs-testing && yarn smoke:cubesql
 
   native:
-    needs: [lint,unit,unit_legacy]
+    needs: [lint]
     runs-on: ${{ matrix.os-version }}
     timeout-minutes: 60
-    name: Build ${{ matrix.os-version }} ${{ matrix.node-version }}
+    name: Build ${{ matrix.os-version }} ${{ matrix.node-version }} with Python ${{ matrix.python-version }}
 
     strategy:
       matrix:
         # We dont need to test under all versions, we do it under linux
         node-version: [16.x]
         os-version: [windows-2019, macos-11]
+        python-version: ["3.9", "3.10", "3.11", "false"]
       fail-fast: false
 
     steps:
@@ -221,8 +229,9 @@ jobs:
           components: rustfmt
       - name: Install Python
         uses: actions/setup-python@v4
+        if: (matrix.python-version != 'false')
         with:
-          python-version: '3.11'
+          python-version: ${{ matrix.python-version }}
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v3
         with:
@@ -252,8 +261,15 @@ jobs:
           command: yarn install --frozen-lockfile
       - name: Lerna tsc
         run: yarn tsc
-      - name: Build native
+      - name: Build native (fallback)
+        if: (matrix.python-version == 'false')
         env:
           CUBESQL_STREAM_MODE: true
           CUBEJS_NATIVE_INTERNAL_DEBUG: true
         run: cd packages/cubejs-backend-native && yarn run native:build && yarn run test:unit
+      - name: Build native (with Python)
+        if: (matrix.python-version != 'false')
+        env:
+          CUBESQL_STREAM_MODE: true
+          CUBEJS_NATIVE_INTERNAL_DEBUG: true
+        run: cd packages/cubejs-backend-native && yarn run native:build-debug-python && yarn run test:unit

packages/cubejs-backend-native/Cargo.toml

@@ -32,3 +32,7 @@ convert_case = "0.4"
 version = "=0.10.0"
 default-features = false
 features = ["napi-1", "napi-4", "napi-6", "channel-api", "promise-api", "task-api", "proc-macros"]
+
+[features]
+default = []
+python = []

packages/cubejs-backend-native/js/index.ts

@@ -198,6 +198,11 @@ export const setupLogger = (logger: (extra: any) => unknown, logLevel: LogLevel)
     native.setupLogger({logger: wrapNativeFunctionWithChannelCallback(logger), logLevel});
 }
 
+export const isFallbackBuild = (): boolean => {
+    const native = loadNative();
+    return native.isFallbackBuild();
+}
+
 export type SqlInterfaceInstance = { __typename: 'sqlinterfaceinstance' };
 
 export const registerInterface = async (options: SQLInterfaceOptions): Promise<SqlInterfaceInstance> => {
@@ -245,6 +250,10 @@ interface PyConfiguration {
 }
 
 export const pythonLoadConfig = async (context: string, options: { file: string }): Promise<PyConfiguration> => {
+    if (isFallbackBuild()) {
+        throw new Error('Python is not supported in fallback build');
+    }
+
     const native = loadNative();
     const config = await native.pythonLoadConfig(context, options);
 

packages/cubejs-backend-native/package.json

@@ -15,6 +15,8 @@
     "native:build": "tsc && cargo-cp-artifact -a cdylib cubejs-native index.node -- cargo build --message-format=json-render-diagnostics",
     "native:build-debug": "npm run native:build --",
     "native:build-release": "npm run native:build -- --release",
+    "native:build-debug-python": "npm run native:build -- --features python",
+    "native:build-release-python": "npm run native:build -- --release --features python",
     "install": "node-pre-gyp install || echo 'Your system is not supported by @cubejs-backend/native, some feature will be unavailable.'",
     "upload-binary-cross": "mkdir -p native && cp index.node native/index.node && node-pre-gyp package --target_arch=$PACKAGE_TARGET_ARCH --target_platform=$PACKAGE_TARGET_PLATFORM --target_libc=$PACKAGE_TARGET_LIBC && node-pre-gyp-github publish",
     "upload-binary": "mkdir -p native && cp index.node native/index.node && node-pre-gyp package && node-pre-gyp-github publish",

packages/cubejs-backend-native/src/lib.rs

@@ -4,6 +4,7 @@ mod auth;
 mod channel;
 mod config;
 mod logger;
+#[cfg(feature = "python")]
 mod python;
 mod stream;
 mod transport;
@@ -13,14 +14,12 @@ use once_cell::sync::OnceCell;
 
 use std::sync::Arc;
 
-use crate::python::CubeConfigPy;
 use auth::NodeBridgeAuthService;
 use config::NodeConfig;
 use cubesql::{config::CubeServices, telemetry::ReportingLogger};
 use log::Level;
 use logger::NodeBridgeLogger;
 use neon::prelude::*;
-use pyo3::prelude::*;
 use simple_logger::SimpleLogger;
 use tokio::runtime::{Builder, Runtime};
 use transport::NodeBridgeTransport;
@@ -48,19 +47,6 @@ fn runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&'static Runtime> {
     })
 }
 
-fn py_runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&()> {
-    static PY_RUNTIME: OnceCell<()> = OnceCell::new();
-
-    let runtime = runtime(cx)?;
-
-    PY_RUNTIME.get_or_try_init(|| {
-        pyo3::prepare_freethreaded_python();
-        pyo3_asyncio::tokio::init_with_runtime(runtime).unwrap();
-
-        Ok(())
-    })
-}
-
 fn setup_logger(mut cx: FunctionContext) -> JsResult<JsUndefined> {
     let options = cx.argument::<JsObject>(0)?;
     let cube_logger = options
@@ -202,50 +188,25 @@ fn shutdown_interface(mut cx: FunctionContext) -> JsResult<JsPromise> {
     Ok(promise)
 }
 
-fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
-    let config_file_content = cx.argument::<JsString>(0)?.value(&mut cx);
-
-    let (deferred, promise) = cx.promise();
-    let channel = cx.channel();
-
-    py_runtime(&mut cx)?;
-
-    let conf_res = Python::with_gil(|py| -> PyResult<CubeConfigPy> {
-        let cube_conf_code = include_str!(concat!(
-            env!("CARGO_MANIFEST_DIR"),
-            "/python/cube/src/conf/__init__.py"
-        ));
-        PyModule::from_code(py, cube_conf_code, "__init__.py", "cube.conf")?;
-
-        let config_module = PyModule::from_code(py, &config_file_content, "config.py", "")?;
-        let settings_py = config_module.getattr("settings")?;
-
-        let mut cube_conf = CubeConfigPy::new();
-
-        for attr_name in cube_conf.get_static_attrs() {
-            cube_conf.static_from_attr(settings_py, attr_name)?;
-        }
-
-        cube_conf.apply_dynamic_functions(settings_py)?;
-
-        Ok(cube_conf)
-    });
-
-    deferred.settle_with(&channel, move |mut cx| match conf_res {
-        Ok(c) => c.to_object(&mut cx),
-        Err(err) => cx.throw_error(format!("Python error: {}", err)),
-    });
+fn is_fallback_build(mut cx: FunctionContext) -> JsResult<JsBoolean> {
+    #[cfg(feature = "python")]
+    {
+        return Ok(JsBoolean::new(&mut cx, false));
+    }
 
-    Ok(promise)
+    #[allow(unreachable_code)]
+    Ok(JsBoolean::new(&mut cx, true))
 }
 
 #[neon::main]
 fn main(mut cx: ModuleContext) -> NeonResult<()> {
     cx.export_function("setupLogger", setup_logger)?;
     cx.export_function("registerInterface", register_interface)?;
     cx.export_function("shutdownInterface", shutdown_interface)?;
+    cx.export_function("isFallbackBuild", is_fallback_build)?;
 
-    cx.export_function("pythonLoadConfig", python_load_config)?;
+    #[cfg(feature = "python")]
+    python::python_register_module(cx)?;
 
     Ok(())
 }

packages/cubejs-backend-native/src/python/entry.rs

@@ -0,0 +1,61 @@
+use crate::python::cube_config::CubeConfigPy;
+use crate::runtime;
+use neon::prelude::*;
+use once_cell::sync::OnceCell;
+use pyo3::prelude::*;
+
+fn py_runtime<'a, C: Context<'a>>(cx: &mut C) -> NeonResult<&()> {
+    static PY_RUNTIME: OnceCell<()> = OnceCell::new();
+
+    let runtime = runtime(cx)?;
+
+    PY_RUNTIME.get_or_try_init(|| {
+        pyo3::prepare_freethreaded_python();
+        pyo3_asyncio::tokio::init_with_runtime(runtime).unwrap();
+
+        Ok(())
+    })
+}
+
+fn python_load_config(mut cx: FunctionContext) -> JsResult<JsPromise> {
+    let config_file_content = cx.argument::<JsString>(0)?.value(&mut cx);
+
+    let (deferred, promise) = cx.promise();
+    let channel = cx.channel();
+
+    py_runtime(&mut cx)?;
+
+    let conf_res = Python::with_gil(|py| -> PyResult<CubeConfigPy> {
+        let cube_conf_code = include_str!(concat!(
+            env!("CARGO_MANIFEST_DIR"),
+            "/python/cube/src/conf/__init__.py"
+        ));
+        PyModule::from_code(py, cube_conf_code, "__init__.py", "cube.conf")?;
+
+        let config_module = PyModule::from_code(py, &config_file_content, "config.py", "")?;
+        let settings_py = config_module.getattr("settings")?;
+
+        let mut cube_conf = CubeConfigPy::new();
+
+        for attr_name in cube_conf.get_static_attrs() {
+            cube_conf.static_from_attr(settings_py, attr_name)?;
+        }
+
+        cube_conf.apply_dynamic_functions(settings_py)?;
+
+        Ok(cube_conf)
+    });
+
+    deferred.settle_with(&channel, move |mut cx| match conf_res {
+        Ok(c) => c.to_object(&mut cx),
+        Err(err) => cx.throw_error(format!("Python error: {}", err)),
+    });
+
+    Ok(promise)
+}
+
+pub fn python_register_module(mut cx: ModuleContext) -> NeonResult<()> {
+    cx.export_function("pythonLoadConfig", python_load_config)?;
+
+    Ok(())
+}

packages/cubejs-backend-native/src/python/mod.rs

@@ -1,4 +1,5 @@
-mod cross;
-mod cube_config;
+pub(crate) mod cross;
+pub(crate) mod cube_config;
+mod entry;
 
-pub use cube_config::*;
+pub use entry::python_register_module;

packages/cubejs-backend-native/test/python.test.ts

@@ -2,9 +2,10 @@ import fs from 'fs/promises';
 import path from 'path';
 
 import * as native from '../js';
-import * as assert from 'assert';
 
-describe('Python', () => {
+const suite = native.isFallbackBuild() ? xdescribe : describe;
+
+suite('Python', () => {
   async function loadConfigurationFile() {
       const content = await fs.readFile(path.join(process.cwd(), 'test', 'config.py'), 'utf8');
       console.log('content', {

packages/cubejs-backend-shared/src/cli.ts

@@ -22,3 +22,13 @@ export const displayCLIError = async (text: string, pkg: string) => {
 export const displayCLIWarning = (message: string) => {
   console.log(`${color.yellow('Warning.')} ${message}`);
 };
+
+const warningOnce = new Map();
+
+export const displayCLIWarningOnce = (key: string, message: string) => {
+  if (!warningOnce.has(key)) {
+    warningOnce.set(key, 1);
+
+    displayCLIWarning(message);
+  }
+};

packages/cubejs-backend-shared/src/env.ts

@@ -1,7 +1,7 @@
 /* eslint-disable no-restricted-syntax */
 import { get } from 'env-var';
 import { displayCLIWarning } from './cli';
-import { detectLibc } from './platform';
+import { isNativeSupported } from './platform';
 
 export class InvalidConfiguration extends Error {
   public constructor(key: string, value: any, description: string) {
@@ -1543,11 +1543,15 @@ const variables: Record<string, (...args: any) => any> = {
       .asBoolStrict();
 
     if (isDevMode) {
-      if (process.platform === 'linux' && detectLibc() === 'musl') {
-        return undefined;
-      }
+      if (isNativeSupported()) {
+        return 15432;
+      } else {
+        displayCLIWarning(
+          'Native module is not supported on your platform. Please use official docker image as a recommended way'
+        );
 
-      return 15432;
+        return false;
+      }
     }
 
     return undefined;