feat(schema-compiler): Use SWC compiler instead of Babel for js models transpilation

.dockerignore

@@ -10,6 +10,18 @@
 !rust/cubestore/package.json
 !rust/cubestore/bin
 !rust/cubesql/package.json
+!rust/check-dup-prop-transpiler-swc-plugin/package.json
+!rust/cube-prop-ctx-transpiler-swc-plugin/package.json
+!rust/import-export-transpiler-swc-plugin/package.json
+!rust/validation-transpiler-swc-plugin/package.json
+!rust/check-dup-prop-transpiler-swc-plugin/Cargo.toml
+!rust/cube-prop-ctx-transpiler-swc-plugin/Cargo.toml
+!rust/import-export-transpiler-swc-plugin/Cargo.toml
+!rust/validation-transpiler-swc-plugin/Cargo.toml
+!rust/check-dup-prop-transpiler-swc-plugin/target/wasm32-wasip1/release/*.wasm
+!rust/cube-prop-ctx-transpiler-swc-plugin/target/wasm32-wasip1/release/*.wasm
+!rust/import-export-transpiler-swc-plugin/target/wasm32-wasip1/release/*.wasm
+!rust/validation-transpiler-swc-plugin/target/wasm32-wasip1/release/*.wasm
 
 # Ignoring builds for native from local machime to protect a problem with different architecture
 packages/cubejs-backend-native/index.node
@@ -21,3 +33,8 @@ packages/*/dist/
 packages/*/coverage/
 # Other
 packages/cubejs-server-core/playground/
+# Rust swc caches
+rust/check-dup-prop-transpiler-swc-plugin/target
+rust/cube-prop-ctx-transpiler-swc-plugin/target
+rust/import-export-transpiler-swc-plugin/target
+rust/validation-transpiler-swc-plugin/target

.github/workflows/cloud.yml

@@ -75,7 +75,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:

.github/workflows/drivers-tests.yml

@@ -108,7 +108,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:

.github/workflows/master.yml

@@ -59,7 +59,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: x86_64-unknown-linux-gnu
+          target: x86_64-unknown-linux-gnu wasm32-wasip1
           cache: false
       - uses: Swatinem/rust-cache@v2
         with:

.github/workflows/post-release.yml

@@ -36,7 +36,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: x86_64-unknown-linux-gnu
+          target: x86_64-unknown-linux-gnu wasm32-wasip1
           cache: false
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4

.github/workflows/publish.yml

@@ -26,6 +26,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Node.js 20.x
         uses: actions/setup-node@v4
         with:
@@ -99,7 +100,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -189,7 +190,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
       - name: Install Python
         uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
@@ -264,6 +265,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Python
         uses: actions/setup-python@v5
         if: (matrix.python-version != 'fallback')
@@ -618,7 +620,7 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: nightly-2024-01-29
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
@@ -705,7 +707,7 @@ jobs:
         uses: actions-rust-lang/setup-rust-toolchain@v1
         with:
           toolchain: nightly-2024-01-29
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt

.github/workflows/push.yml

@@ -13,6 +13,10 @@ on:
       - 'rust/cubeorchestrator/**'
       - 'rust/cubeshared/**'
       - 'rust/cubesqlplanner/**'
+      - 'rust/check-dup-prop-transpiler-swc-plugin/**'
+      - 'rust/cube-prop-ctx-transpiler-swc-plugin/**'
+      - 'rust/import-export-transpiler-swc-plugin/**'
+      - 'rust/validation-transpiler-swc-plugin/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
@@ -32,6 +36,10 @@ on:
       - 'rust/cubeorchestrator/**'
       - 'rust/cubeshared/**'
       - 'rust/cubesqlplanner/**'
+      - 'rust/check-dup-prop-transpiler-swc-plugin/**'
+      - 'rust/cube-prop-ctx-transpiler-swc-plugin/**'
+      - 'rust/import-export-transpiler-swc-plugin/**'
+      - 'rust/validation-transpiler-swc-plugin/**'
       - '.eslintrc.js'
       - '.prettierrc'
       - 'package.json'
@@ -76,6 +84,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -111,6 +120,8 @@ jobs:
         run: yarn tsc
       - name: Build client
         run: yarn build
+      - name: Build transpilers swc plugins
+        run: yarn lerna run build-swc-plugin
       - name: Build cubejs-backend-native (with Python)
         run: yarn run native:build-release-python
         working-directory: ./packages/cubejs-backend-native
@@ -139,6 +150,18 @@ jobs:
 #      - name: Cargo test cubesqlplanner
 #        run: |
 #          cargo test --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -j 1
+      - name: Cargo test check-dup-prop-transpiler-swc-plugin
+        run: |
+          cargo test --manifest-path rust/check-dup-prop-transpiler-swc-plugin/Cargo.toml -j 1
+      - name: Cargo test cube-prop-ctx-transpiler-swc-plugin
+        run: |
+          cargo test --manifest-path rust/cube-prop-ctx-transpiler-swc-plugin/Cargo.toml -j 1
+      - name: Cargo test import-export-transpiler-swc-plugin
+        run: |
+          cargo test --manifest-path rust/import-export-transpiler-swc-plugin/Cargo.toml -j 1
+      - name: Cargo test validation-transpiler-swc-plugin
+        run: |
+          cargo test --manifest-path rust/validation-transpiler-swc-plugin/Cargo.toml -j 1
 
   lint:
     runs-on: ubuntu-20.04
@@ -156,6 +179,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Node.js 20.x
         uses: actions/setup-node@v4
         with:
@@ -202,6 +226,18 @@ jobs:
 #      - name: Cargo fmt cubesqlplanner
 #        run: |
 #          cargo fmt --manifest-path rust/cubesqlplanner/cubesqlplanner/Cargo.toml -- --check
+      - name: Cargo fmt check-dup-prop-transpiler-swc-plugin
+        run: |
+          cargo fmt --manifest-path rust/check-dup-prop-transpiler-swc-plugin/Cargo.toml -- --check
+      - name: Cargo fmt cube-prop-ctx-transpiler-swc-plugin
+        run: |
+          cargo fmt --manifest-path rust/cube-prop-ctx-transpiler-swc-plugin/Cargo.toml -- --check
+      - name: Cargo fmt import-export-transpiler-swc-plugin
+        run: |
+          cargo fmt --manifest-path rust/import-export-transpiler-swc-plugin/Cargo.toml -- --check
+      - name: Cargo fmt validation-transpiler-swc-plugin
+        run: |
+          cargo fmt --manifest-path rust/validation-transpiler-swc-plugin/Cargo.toml -- --check
 
   build:
     runs-on: ubuntu-20.04
@@ -219,6 +255,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Node.js 20.x
         uses: actions/setup-node@v4
         with:
@@ -418,6 +455,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
+          target: wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -484,8 +522,11 @@ jobs:
       matrix:
         node-version: [ 20.x ]
         python-version: [ 3.11 ]
+        transpile-worker-threads: [false, true]
       fail-fast: false
 
+    env:
+      CUBEJS_TRANSPILATION_WORKER_THREADS: ${{ matrix.transpile-worker-threads }}
     steps:
       - name: Maximize build space (disk space limitations)
         run: |
@@ -509,6 +550,14 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          toolchain: nightly-2024-07-15
+          # override: true # this is by default on
+          rustflags: ""
+          components: rustfmt
+          target: wasm32-wasip1
       - name: Install Python
         uses: actions/setup-python@v5
         with:
@@ -540,6 +589,8 @@ jobs:
         uses: GoodManWEN/oracle-client-action@main
       - name: Build client
         run: yarn build
+      - name: Build transpilers swc plugins
+        run: yarn lerna run build-swc-plugin
       - name: Build cubejs-backend-native (with Python)
         run: yarn run native:build-release-python
         working-directory: ./packages/cubejs-backend-native
@@ -645,7 +696,7 @@ jobs:
           # override: true # this is by default on
           rustflags: ""
           components: rustfmt
-          target: ${{ matrix.target }}
+          target: ${{ matrix.target }} wasm32-wasip1
       - name: Install Node.js ${{ matrix.node-version }}
         uses: actions/setup-node@v4
         with:
@@ -686,6 +737,8 @@ jobs:
         run: yarn build
       - name: Lerna tsc
         run: yarn tsc
+      - name: Build transpilers swc plugins
+        run: yarn lerna run build-swc-plugin
       - name: Build cubejs-backend-native (without Python)
         run: yarn run native:build-release
         working-directory: ./packages/cubejs-backend-native

.gitignore

@@ -24,3 +24,4 @@ rust/cubesql/profile.json
 .cubestore
 .env
 .vimspector.json
+.swc

packages/cubejs-docker/dev.Dockerfile

@@ -18,19 +18,14 @@ ENV CARGO_HOME=/usr/local/cargo
 ENV PATH=/usr/local/cargo/bin:$PATH
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
-    sh -s -- --profile minimal --default-toolchain nightly-2022-03-08 -y
+    sh -s -- --profile minimal --default-toolchain nightly-2022-03-08 -y --target wasm32-wasip1
 
 ENV CUBESTORE_SKIP_POST_INSTALL=true
 ENV NODE_ENV=development
 
 WORKDIR /cubejs
 
-COPY package.json .
-COPY lerna.json .
-COPY yarn.lock .
-COPY tsconfig.base.json .
-COPY rollup.config.js .
-COPY packages/cubejs-linter packages/cubejs-linter
+COPY package.json lerna.json yarn.lock tsconfig.base.json rollup.config.js ./
 
 # Backend
 COPY rust/cubesql/package.json rust/cubesql/package.json
@@ -88,6 +83,11 @@ COPY packages/cubejs-client-ngx/package.json packages/cubejs-client-ngx/package.
 COPY packages/cubejs-client-ws-transport/package.json packages/cubejs-client-ws-transport/package.json
 COPY packages/cubejs-playground/package.json packages/cubejs-playground/package.json
 
+COPY rust/check-dup-prop-transpiler-swc-plugin/package.json rust/check-dup-prop-transpiler-swc-plugin/Cargo.toml rust/check-dup-prop-transpiler-swc-plugin/
+COPY rust/cube-prop-ctx-transpiler-swc-plugin/package.json rust/cube-prop-ctx-transpiler-swc-plugin/Cargo.toml rust/cube-prop-ctx-transpiler-swc-plugin/
+COPY rust/import-export-transpiler-swc-plugin/package.json rust/import-export-transpiler-swc-plugin/Cargo.toml rust/import-export-transpiler-swc-plugin/
+COPY rust/validation-transpiler-swc-plugin/package.json rust/validation-transpiler-swc-plugin/Cargo.toml rust/validation-transpiler-swc-plugin/
+
 RUN yarn policies set-version v1.22.22
 # Yarn v1 uses aggressive timeouts with summing time spending on fs, https://github.com/yarnpkg/yarn/issues/4890
 RUN yarn config set network-timeout 120000 -g
@@ -166,6 +166,11 @@ COPY packages/cubejs-client-ngx/ packages/cubejs-client-ngx/
 COPY packages/cubejs-client-ws-transport/ packages/cubejs-client-ws-transport/
 COPY packages/cubejs-playground/ packages/cubejs-playground/
 
+COPY rust/check-dup-prop-transpiler-swc-plugin/ rust/check-dup-prop-transpiler-swc-plugin/
+COPY rust/cube-prop-ctx-transpiler-swc-plugin/ rust/cube-prop-ctx-transpiler-swc-plugin/
+COPY rust/import-export-transpiler-swc-plugin/ rust/import-export-transpiler-swc-plugin/
+COPY rust/validation-transpiler-swc-plugin/ rust/validation-transpiler-swc-plugin/
+
 RUN yarn build
 RUN yarn lerna run build
 

packages/cubejs-docker/testing-drivers.Dockerfile

@@ -68,6 +68,11 @@ COPY packages/cubejs-ksql-driver/package.json packages/cubejs-ksql-driver/packag
 COPY packages/cubejs-dbt-schema-extension/package.json packages/cubejs-dbt-schema-extension/package.json
 COPY packages/cubejs-jdbc-driver/package.json packages/cubejs-jdbc-driver/package.json
 
+COPY rust/check-dup-prop-transpiler-swc-plugin/package.json rust/check-dup-prop-transpiler-swc-plugin/package.json
+COPY rust/cube-prop-ctx-transpiler-swc-plugin/package.json rust/cube-prop-ctx-transpiler-swc-plugin/package.json
+COPY rust/import-export-transpiler-swc-plugin/package.json rust/import-export-transpiler-swc-plugin/package.json
+COPY rust/validation-transpiler-swc-plugin/package.json rust/validation-transpiler-swc-plugin/package.json
+
 # We dont need client libraries
 #COPY packages/cubejs-templates/package.json packages/cubejs-templates/package.json
 #COPY packages/cubejs-client-core/package.json packages/cubejs-client-core/package.json
@@ -147,6 +152,11 @@ COPY packages/cubejs-dbt-schema-extension/ packages/cubejs-dbt-schema-extension/
 COPY packages/cubejs-jdbc-driver/ packages/cubejs-jdbc-driver/
 COPY packages/cubejs-databricks-jdbc-driver/ packages/cubejs-databricks-jdbc-driver/
 
+COPY rust/check-dup-prop-transpiler-swc-plugin/ rust/check-dup-prop-transpiler-swc-plugin/
+COPY rust/cube-prop-ctx-transpiler-swc-plugin/ rust/cube-prop-ctx-transpiler-swc-plugin/
+COPY rust/import-export-transpiler-swc-plugin/ rust/import-export-transpiler-swc-plugin/
+COPY rust/validation-transpiler-swc-plugin/ rust/validation-transpiler-swc-plugin/
+
 # We dont need client libraries
 #COPY packages/cubejs-templates/ packages/cubejs-templates/
 #COPY packages/cubejs-client-core/ packages/cubejs-client-core/

packages/cubejs-schema-compiler/package.json

@@ -42,6 +42,11 @@
     "@babel/types": "^7.24",
     "@cubejs-backend/native": "1.2.5",
     "@cubejs-backend/shared": "1.2.5",
+    "@cubejs-backend/check-dup-prop-transpiler-swc-plugin": "1.2.5",
+    "@cubejs-backend/cube-prop-ctx-transpiler-swc-plugin": "1.2.5",
+    "@cubejs-backend/import-export-transpiler-swc-plugin": "1.2.5",
+    "@cubejs-backend/validation-transpiler-swc-plugin": "1.2.5",
+    "@swc/core": "^1.10.14",
     "antlr4ts": "0.5.0-alpha.4",
     "camelcase": "^6.2.0",
     "cron-parser": "^4.9.0",