Skip to content

Commit 4f24eb1

Browse files
ovrovr
authored
chore(ci): PR workflow - build native once for all jobs (speedup builds) (#9388)
1 parent cb61037 commit 4f24eb1

File tree

1 file changed

+93
-52
lines changed

1 file changed

+93
-52
lines changed

‎.github/workflows/push.yml‎

Lines changed: 93 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,21 @@ on:
4141

4242
env:
4343
CUBEJS_TESSERACT_ORCHESTRATOR: true
44+
# Current user version for Python. Should be aligned between build-native & docker-dev
45+
PYTHON_VERSION_CURRENT: 3.11
4446

4547
jobs:
4648
unit:
4749
runs-on: ubuntu-24.04
4850
timeout-minutes: 60
49-
needs: latest-tag-sha
51+
needs: [latest-tag-sha, build-native-linux]
5052
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
5153

5254
strategy:
5355
matrix:
5456
# Current docker version + next LTS
5557
node-version: [20.x, 22.x]
58+
# Don't forget to update build-native-release
5659
python-version: [3.11]
5760
transpile-worker-threads: [false, true]
5861
fail-fast: false
@@ -69,13 +72,11 @@ jobs:
6972
with:
7073
# pulls all commits (needed for codecov)
7174
fetch-depth: 2
72-
- name: Install Rust
73-
uses: actions-rust-lang/setup-rust-toolchain@v1
75+
- name: Download backend-native artifact
76+
uses: actions/download-artifact@v4
7477
with:
75-
toolchain: 1.84.1
76-
# override: true # this is by default on
77-
rustflags: ""
78-
components: rustfmt
78+
name: "native-linux-x64-glibc-${{ matrix.python-version }}.node"
79+
path: ./packages/cubejs-backend-native/
7980
- name: Install Node.js ${{ matrix.node-version }}
8081
uses: actions/setup-node@v4
8182
with:
@@ -111,11 +112,6 @@ jobs:
111112
run: yarn tsc
112113
- name: Build client
113114
run: yarn build
114-
- name: Build cubejs-backend-native (with Python)
115-
run: yarn run native:build-release-python
116-
working-directory: ./packages/cubejs-backend-native
117-
env:
118-
PYO3_PYTHON: python${{ matrix.python-version }}
119115
- name: Lerna test
120116
run: yarn lerna run --concurrency 1 --stream --no-prefix unit
121117
# - uses: codecov/codecov-action@v1
@@ -304,8 +300,61 @@ jobs:
304300
path: ./rust/cubestore/target/release/cubestored
305301
retention-days: 5
306302

303+
build-native-linux:
304+
runs-on: ubuntu-24.04
305+
needs: [latest-tag-sha]
306+
timeout-minutes: 60
307+
name: "Build native linux (Python: ${{ matrix.python-version }})"
308+
strategy:
309+
matrix:
310+
# Don't forget to align with PYTHON_VERSION_CURRENT
311+
python-version: [ 3.11 ]
312+
fail-fast: false
313+
container:
314+
image: cubejs/rust-cross:x86_64-unknown-linux-gnu-15082024-python-${{ matrix.python-version }}
315+
steps:
316+
- name: Checkout
317+
uses: actions/checkout@v4
318+
- name: Install Rust
319+
uses: actions-rust-lang/setup-rust-toolchain@v1
320+
with:
321+
toolchain: 1.84.1
322+
# override: true # this is by default on
323+
rustflags: ""
324+
components: rustfmt
325+
target: x86_64-unknown-linux-gnu
326+
cache: false
327+
- uses: Swatinem/rust-cache@v2
328+
with:
329+
workspaces: ./packages/cubejs-backend-native
330+
key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
331+
shared-key: native-${{ runner.OS }}-x86_64-unknown-linux-gnu
332+
- name: Install Node.js 20
333+
uses: actions/setup-node@v4
334+
with:
335+
node-version: 20
336+
- name: Install Yarn
337+
run: npm install -g yarn
338+
- name: Set Yarn version
339+
run: yarn policies set-version v1.22.22
340+
# We don't need to install all yarn deps to build native
341+
- name: Install cargo-cp-artifact
342+
run: npm install -g [email protected]
343+
- name: Build native (with Python)
344+
env:
345+
PYO3_PYTHON: python${{ env.PYTHON_VERSION_CURRENT }}
346+
CARGO_BUILD_TARGET: x86_64-unknown-linux-gnu
347+
working-directory: ./packages/cubejs-backend-native
348+
run: yarn run native:build-release-python
349+
- name: Store build artifact for dev image
350+
uses: actions/upload-artifact@v4
351+
with:
352+
name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
353+
path: ./packages/cubejs-backend-native/index.node
354+
overwrite: true
355+
307356
integration-cubestore:
308-
needs: [latest-tag-sha, build-cubestore]
357+
needs: [latest-tag-sha, build-cubestore, build-native-linux]
309358
runs-on: ubuntu-24.04
310359
timeout-minutes: 60
311360
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
@@ -334,17 +383,14 @@ jobs:
334383
df -h
335384
- name: Checkout
336385
uses: actions/checkout@v4
337-
- name: Install Rust
338-
uses: actions-rust-lang/setup-rust-toolchain@v1
339-
with:
340-
toolchain: 1.84.1
341-
# override: true # this is by default on
342-
rustflags: ""
343-
components: rustfmt
344386
- name: Install Node.js ${{ matrix.node-version }}
345387
uses: actions/setup-node@v4
346388
with:
347389
node-version: ${{ matrix.node-version }}
390+
- name: Install Python
391+
uses: actions/setup-python@v5
392+
with:
393+
python-version: ${{ env.PYTHON_VERSION_CURRENT }}
348394
- name: Get yarn cache directory path
349395
id: yarn-cache-dir-path
350396
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -370,10 +416,12 @@ jobs:
370416
command: yarn install --frozen-lockfile
371417
- name: Lerna tsc
372418
run: yarn tsc
373-
- name: Build cubejs-backend-native (without Python)
374-
run: yarn run native:build-release
375-
working-directory: ./packages/cubejs-backend-native
376-
- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
419+
- name: Download backend-native artifact
420+
uses: actions/download-artifact@v4
421+
with:
422+
name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
423+
path: ./packages/cubejs-backend-native/
424+
- name: Download cubestored artifact
377425
uses: actions/download-artifact@v4
378426
with:
379427
path: ./rust/cubestore/target/release/
@@ -388,7 +436,7 @@ jobs:
388436
yarn lerna run --concurrency 1 --stream --no-prefix integration:cubestore
389437
390438
integration:
391-
needs: [unit, lint, latest-tag-sha]
439+
needs: [unit, lint, latest-tag-sha, build-native-linux]
392440
runs-on: ubuntu-24.04
393441
timeout-minutes: 60
394442
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
@@ -411,13 +459,22 @@ jobs:
411459
steps:
412460
- name: Checkout
413461
uses: actions/checkout@v4
462+
- name: Download backend-native artifact
463+
uses: actions/download-artifact@v4
464+
with:
465+
name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
466+
path: ./packages/cubejs-backend-native/
414467
- name: Install Rust
415468
uses: actions-rust-lang/setup-rust-toolchain@v1
416469
with:
417470
toolchain: 1.84.1
418471
# override: true # this is by default on
419472
rustflags: ""
420473
components: rustfmt
474+
- name: Install Python
475+
uses: actions/setup-python@v5
476+
with:
477+
python-version: ${{ env.PYTHON_VERSION_CURRENT }}
421478
- name: Install Node.js ${{ matrix.node-version }}
422479
uses: actions/setup-node@v4
423480
with:
@@ -447,9 +504,6 @@ jobs:
447504
command: yarn install --frozen-lockfile
448505
- name: Lerna tsc
449506
run: yarn tsc
450-
- name: Build cubejs-backend-native (without Python)
451-
run: yarn run native:build-release
452-
working-directory: ./packages/cubejs-backend-native
453507
- name: Run Integration tests for ${{ matrix.db }} matrix
454508
uses: nick-fields/retry@v3
455509
with:
@@ -475,7 +529,7 @@ jobs:
475529
DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN: ${{ secrets.DRIVERS_TESTS_DREMIO_CUBEJS_DB_DREMIO_AUTH_TOKEN }}
476530

477531
integration-smoke:
478-
needs: [ latest-tag-sha, build-cubestore ]
532+
needs: [latest-tag-sha, build-cubestore, build-native-linux]
479533
runs-on: ubuntu-24.04
480534
timeout-minutes: 90
481535
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
@@ -505,6 +559,11 @@ jobs:
505559
df -h
506560
- name: Checkout
507561
uses: actions/checkout@v4
562+
- name: Download backend-native artifact
563+
uses: actions/download-artifact@v4
564+
with:
565+
name: "native-linux-x64-glibc-${{ matrix.python-version }}.node"
566+
path: ./packages/cubejs-backend-native/
508567
- name: Install Node.js ${{ matrix.node-version }}
509568
uses: actions/setup-node@v4
510569
with:
@@ -540,11 +599,6 @@ jobs:
540599
uses: GoodManWEN/oracle-client-action@main
541600
- name: Build client
542601
run: yarn build
543-
- name: Build cubejs-backend-native (with Python)
544-
run: yarn run native:build-release-python
545-
working-directory: ./packages/cubejs-backend-native
546-
env:
547-
PYO3_PYTHON: python${{ matrix.python-version }}
548602
- name: Lerna tsc
549603
run: yarn tsc
550604
- name: Download cubestored-x86_64-unknown-linux-gnu-release artifact
@@ -598,7 +652,7 @@ jobs:
598652
OUT: ${{ steps.get-tag.outputs.sha }}
599653

600654
docker-dev:
601-
needs: [latest-tag-sha]
655+
needs: [latest-tag-sha, build-native-linux]
602656
if: (needs['latest-tag-sha'].outputs.sha != github.sha)
603657
name: Build & Test :dev for ${{ matrix.name }} without pushing
604658
runs-on: ubuntu-22.04
@@ -638,14 +692,11 @@ jobs:
638692
df -h
639693
- name: Checkout
640694
uses: actions/checkout@v4
641-
- name: Install Rust
642-
uses: actions-rust-lang/setup-rust-toolchain@v1
695+
- name: Download backend-native artifact
696+
uses: actions/download-artifact@v4
643697
with:
644-
toolchain: 1.84.1
645-
# override: true # this is by default on
646-
rustflags: ""
647-
components: rustfmt
648-
target: ${{ matrix.target }}
698+
name: "native-linux-x64-glibc-${{ env.PYTHON_VERSION_CURRENT }}.node"
699+
path: ./packages/cubejs-backend-native/
649700
- name: Install Node.js ${{ matrix.node-version }}
650701
uses: actions/setup-node@v4
651702
with:
@@ -654,13 +705,6 @@ jobs:
654705
run: npm install -g yarn
655706
- name: Set Yarn version
656707
run: yarn policies set-version v1.22.22
657-
- name: Install cargo-cp-artifact
658-
run: npm install -g [email protected]
659-
- uses: Swatinem/rust-cache@v2
660-
with:
661-
workspaces: ./packages/cubejs-backend-native
662-
key: native-${{ runner.OS }}-${{ matrix.target }}
663-
shared-key: native-${{ runner.OS }}-${{ matrix.target }}
664708
- name: Get yarn cache directory path
665709
id: yarn-cache-dir-path
666710
run: echo "dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT"
@@ -686,9 +730,6 @@ jobs:
686730
run: yarn build
687731
- name: Lerna tsc
688732
run: yarn tsc
689-
- name: Build cubejs-backend-native (without Python)
690-
run: yarn run native:build-release
691-
working-directory: ./packages/cubejs-backend-native
692733
- name: Set up QEMU
693734
uses: docker/setup-qemu-action@v3
694735
# current .dockerignore prevents use of native build

0 commit comments

Comments
 (0)