Skip to content

Commit 12a3b3c

Browse files
phil-oppphil-opp
committed
Add CI
1 parent b9b8e44 commit 12a3b3c

File tree

2 files changed

+224
-0
lines changed

2 files changed

+224
-0
lines changed

‎.github/workflows/node-hub-ci-cd.yml‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
name: node-hub
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- main
8+
pull_request:
9+
types:
10+
- opened
11+
- synchronize
12+
- reopened
13+
- ready_for_review
14+
branches:
15+
- main
16+
release:
17+
types: [published]
18+
19+
jobs:
20+
find-jobs:
21+
runs-on: ubuntu-24.04
22+
name: Find Jobs
23+
outputs:
24+
folders: ${{ steps.jobs.outputs.folders }}
25+
steps:
26+
- uses: actions/checkout@v1
27+
28+
- id: jobs
29+
uses: kmanimaran/list-folder-action@v4
30+
31+
ci:
32+
runs-on: ${{ matrix.platform }}
33+
needs: [find-jobs]
34+
defaults:
35+
run:
36+
working-directory: ${{ matrix.folder }}
37+
strategy:
38+
fail-fast: ${{ github.event_name != 'workflow_dispatch' && !(github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')) }}
39+
matrix:
40+
platform: [ubuntu-24.04, macos-14]
41+
folder: ${{ fromJson(needs.find-jobs.outputs.folders )}}
42+
steps:
43+
- name: Checkout repository
44+
if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
45+
uses: actions/checkout@v4
46+
with:
47+
submodules: true # Make sure to check out the sub-module
48+
49+
- name: Update submodule
50+
if: runner.os == 'Linux'
51+
run: |
52+
git submodule update --init --recursive
53+
git submodule update --remote --recursive
54+
55+
- name: Install system-level dependencies
56+
if: runner.os == 'Linux'
57+
run: |
58+
sudo apt update
59+
sudo apt-get install portaudio19-dev
60+
sudo apt-get install libdav1d-dev nasm libudev-dev
61+
mkdir -p $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib
62+
ln -s /lib/x86_64-linux-gnu/libdav1d.so $HOME/.rustup/toolchains/stable-x86_64-unknown-linux-gnu/lib/rustlib/x86_64-unknown-linux-gnu/lib/libdav1d.so
63+
64+
# Install mingw-w64 cross-compilers
65+
sudo apt install g++-mingw-w64-x86-64 gcc-mingw-w64-x86-64
66+
67+
- name: Install system-level dependencies for MacOS
68+
if: runner.os == 'MacOS' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/')))
69+
run: |
70+
brew install portaudio
71+
brew install dav1d nasm
72+
73+
- name: Set up Python
74+
if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
75+
uses: actions/setup-python@v2
76+
with:
77+
python-version: "3.10"
78+
79+
- name: Install the latest version of uv
80+
uses: astral-sh/setup-uv@v5
81+
82+
- name: Set up Rust
83+
if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
84+
uses: actions-rs/toolchain@v1
85+
with:
86+
toolchain: stable
87+
override: true
88+
89+
- name: Run Linting and Tests
90+
## Run Linting and testing only on Mac for release workflows.
91+
if: runner.os == 'Linux' || github.event_name == 'workflow_dispatch' || (github.event_name == 'release' && startsWith(github.ref, 'refs/tags/'))
92+
env:
93+
GITHUB_EVENT_NAME: ${{ github.event_name }}
94+
MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_PASS }}
95+
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_PASS }}
96+
CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
97+
run: |
98+
chmod +x ../../.github/workflows/node_hub_test.sh
99+
../../.github/workflows/node_hub_test.sh

‎.github/workflows/node_hub_test.sh‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
#!/bin/bash
2+
set -euo
3+
4+
# Check if we are running in a GitHub Actions environment
5+
CI=${GITHUB_ACTIONS:-false}
6+
7+
# List of ignored modules
8+
ignored_folders=("dora-parler" "dora-opus" "dora-internvl" "dora-magma")
9+
10+
# Skip test
11+
skip_test_folders=("dora-internvl" "dora-parler" "dora-keyboard" "dora-microphone" "terminal-input" "dora-magma")
12+
13+
# Get current working directory
14+
dir=$(pwd)
15+
16+
# Get the base name of the directory (without the path)
17+
base_dir=$(basename "$dir")
18+
19+
export GIT_LFS_SKIP_SMUDGE=1
20+
# Large node list requiring space cleanup
21+
large_node=("dora-phi4")
22+
23+
export PYTEST_ADDOPTS="-x"
24+
25+
# Check if the current directory is in the large node list and if we're in the CI environment
26+
if [[ " ${large_node[@]} " =~ " ${base_dir} " ]] && [[ "$CI" == "true" ]]; then
27+
echo "Running cleanup for $base_dir..."
28+
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
29+
# 1.4GB
30+
sudo rm -rf /opt/hostedtoolcache/go || :
31+
# 489MB
32+
sudo rm -rf /opt/hostedtoolcache/PyPy || :
33+
# 376MB
34+
sudo rm -rf /opt/hostedtoolcache/node || :
35+
# Remove Web browser packages
36+
sudo apt purge -y \
37+
firefox \
38+
google-chrome-stable \
39+
microsoft-edge-stable
40+
sudo rm -rf /usr/local/lib/android/
41+
sudo rm -rf /usr/share/dotnet/
42+
sudo rm -rf /opt/ghc/
43+
fi
44+
45+
# Check if the directory name is in the ignored list
46+
if [[ " ${ignored_folders[@]} " =~ " ${base_dir} " ]]; then
47+
echo "Skipping $base_dir as we cannot test it on the CI..."
48+
else
49+
# IF job is mixed rust-python job and is on linux
50+
if [[ -f "Cargo.toml" && -f "pyproject.toml" && "$(uname)" = "Linux" ]]; then
51+
echo "Running build and tests for Rust project in $dir..."
52+
53+
cargo check
54+
cargo clippy
55+
cargo build
56+
cargo test
57+
58+
pip install "maturin[zig, patchelf]"
59+
maturin build --release --compatibility manylinux_2_28 --zig
60+
# If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel on multiple platforms
61+
if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
62+
# Free up ubuntu space
63+
sudo apt-get clean
64+
sudo rm -rf /usr/local/lib/android/
65+
sudo rm -rf /usr/share/dotnet/
66+
sudo rm -rf /opt/ghc/
67+
68+
maturin publish --skip-existing --compatibility manylinux_2_28 --zig
69+
# aarch64-unknown-linux-gnu
70+
rustup target add aarch64-unknown-linux-gnu
71+
maturin publish --target aarch64-unknown-linux-gnu --skip-existing --zig --compatibility manylinux_2_28
72+
73+
# armv7-unknown-linux-musleabihf
74+
rustup target add armv7-unknown-linux-musleabihf
75+
# If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
76+
maturin publish --target armv7-unknown-linux-musleabihf --skip-existing --zig
77+
78+
# x86_64-pc-windows-gnu
79+
rustup target add x86_64-pc-windows-gnu
80+
# If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
81+
maturin publish --target x86_64-pc-windows-gnu --skip-existing
82+
fi
83+
84+
elif [[ -f "Cargo.toml" && -f "pyproject.toml" && "$(uname)" = "Darwin" ]]; then
85+
pip install "maturin[zig, patchelf]"
86+
# aarch64-apple-darwin
87+
maturin build --release
88+
# If GITHUB_EVENT_NAME is release or workflow_dispatch, publish the wheel
89+
if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
90+
maturin publish --skip-existing
91+
fi
92+
93+
elif [[ "$(uname)" = "Linux" ]] || [[ "$CI" == "false" ]]; then
94+
if [ -f "$dir/Cargo.toml" ]; then
95+
echo "Running build and tests for Rust project in $dir..."
96+
cargo check
97+
cargo clippy
98+
cargo build
99+
cargo test
100+
101+
if [ "$GITHUB_EVENT_NAME" == "release" ] || [ "$GITHUB_EVENT_NAME" == "workflow_dispatch" ]; then
102+
cargo publish
103+
fi
104+
else
105+
if [ -f "$dir/pyproject.toml" ]; then
106+
echo "CI: Installing in $dir..."
107+
uv venv --seed -p 3.11
108+
uv pip install .
109+
echo "CI: Running Linting in $dir..."
110+
uv run ruff check .
111+
echo "CI: Running Pytest in $dir..."
112+
# Skip test for some folders
113+
if [[ " ${skip_test_folders[@]} " =~ " ${base_dir} " ]]; then
114+
echo "Skipping tests for $base_dir..."
115+
else
116+
uv run pytest
117+
fi
118+
if [ "${GITHUB_EVENT_NAME:-false}" == "release" ] || [ "${GITHUB_EVENT_NAME:-false}" == "workflow_dispatch" ]; then
119+
uv build
120+
uv publish --check-url https://pypi.org/simple
121+
fi
122+
fi
123+
fi
124+
fi
125+
fi

0 commit comments

Comments
 (0)