Skip to content

Commit a3490a8

Browse files
auxtenauxten
authored
Merge pull request #399 from wudidapaopao/add_musl_linux_wheel_package2
2 parents 11a8c99 + f673710 commit a3490a8

File tree

14 files changed

+858
-12
lines changed

14 files changed

+858
-12
lines changed

‎.github/workflows/build_musllinux_arm64_wheels.yml‎

Lines changed: 362 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,362 @@
1+
name: Build Linux(musllinux) ARM64
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
TAG_NAME:
7+
description: 'Release Version Tag'
8+
required: true
9+
release:
10+
types: [created]
11+
push:
12+
branches:
13+
- main
14+
paths-ignore:
15+
- '**/*.md'
16+
pull_request:
17+
branches:
18+
- main
19+
paths-ignore:
20+
- '**/*.md'
21+
22+
jobs:
23+
build_musllinux_wheels:
24+
name: Build musllinux wheels (Alpine Linux aarch64)
25+
runs-on: GH-Linux-ARM64
26+
steps:
27+
- uses: actions/checkout@v3
28+
with:
29+
fetch-depth: 0
30+
31+
- name: Configure git safe directory
32+
run: |
33+
git config --global --add safe.directory '*'
34+
35+
- name: Update submodules
36+
run: |
37+
git submodule update --init --recursive --jobs 4
38+
39+
- name: Build chdb wheels in container
40+
uses: addnab/docker-run-action@v3
41+
with:
42+
image: quay.io/pypa/musllinux_1_2_aarch64
43+
options: -v ${{ github.workspace }}:/workspace --privileged
44+
run: |
45+
cd /workspace
46+
47+
# Configure git safe directory in container
48+
apk update
49+
apk add --no-cache git python3 py3-pip py3-setuptools
50+
echo "=== Configure git safe directory ==="
51+
git config --global --add safe.directory /workspace
52+
git describe --tags
53+
python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())"
54+
55+
# 1. Check system info
56+
echo "=== Container System Info ==="
57+
echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)"
58+
if [ -f /lib/ld-musl-aarch64.so.1 ]; then
59+
echo "musl libc aarch64"
60+
elif [ -f /lib/libc.musl-aarch64.so.1 ]; then
61+
echo "musl libc aarch64"
62+
else
63+
echo "Not musl libc"
64+
fi
65+
echo "Workspace mounted at: /workspace"
66+
ls -la /workspace
67+
68+
# 2. Install build dependencies
69+
echo "=== Installing build dependencies ==="
70+
apk add --no-cache make build-base openssl-dev zlib-dev \
71+
bzip2-dev readline-dev sqlite-dev wget curl llvm \
72+
ncurses-dev xz-dev tk-dev libxml2-dev \
73+
libffi-dev linux-headers
74+
apk add --no-cache make cmake ccache ninja yasm gawk
75+
apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20
76+
77+
# 3. Scan SQLite vulnerabilities
78+
echo "=== Scanning SQLite vulnerabilities ==="
79+
# Install grype
80+
curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin
81+
grype db update
82+
83+
# Check SQLite vulnerabilities
84+
echo "Scanning SQLite packages for vulnerabilities..."
85+
GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true)
86+
echo "Raw grype output:"
87+
echo "$GRYPE_RAW_OUTPUT"
88+
89+
SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true)
90+
if [ -n "$SQLITE_SCAN_OUTPUT" ]; then
91+
echo "SQLite vulnerabilities found in packages! Build should be reviewed."
92+
echo "SQLite vulnerability details:"
93+
echo "$SQLITE_SCAN_OUTPUT"
94+
else
95+
echo "No SQLite vulnerabilities found"
96+
fi
97+
98+
# 4. Setup Python environments
99+
echo "=== Setting up Python environments ==="
100+
# Setup pyenv
101+
curl https://pyenv.run | bash
102+
export PATH="$HOME/.pyenv/bin:$PATH"
103+
eval "$(pyenv init -)"
104+
105+
# Install Python versions
106+
for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
107+
echo "Installing Python $version"
108+
pyenv install $version:latest
109+
done
110+
pyenv global 3.8 3.9 3.10 3.11 3.12 3.13
111+
112+
# Verify installations
113+
echo "Installed versions:"
114+
pyenv versions
115+
for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
116+
if ! pyenv versions --bare | grep -q "^$version"; then
117+
echo "ERROR: Python $version is not installed!"
118+
exit 1
119+
fi
120+
echo "Python $version is installed"
121+
done
122+
echo "All Python versions verified successfully!"
123+
124+
# Install Rust
125+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
126+
source $HOME/.cargo/env
127+
rustup toolchain install nightly-2025-07-07
128+
rustup component add --toolchain nightly-2025-07-07 rust-src
129+
130+
# Install Python dependencies
131+
for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
132+
echo "Installing dependencies for Python $version"
133+
pyenv shell $version
134+
python -m pip install --upgrade pip
135+
if [ "$version" = "3.8" ]; then
136+
python -m pip install setuptools tox twine psutil wheel
137+
else
138+
python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel
139+
fi
140+
pyenv shell --unset
141+
done
142+
143+
# 5. Build chdb
144+
echo "=== Building chdb ==="
145+
echo "Timestamp: $(date)"
146+
echo "Current directory: $(pwd)"
147+
echo "Available disk space: $(df -h .)"
148+
149+
# Setup clang
150+
echo "Setting up clang compiler..."
151+
ln -sf /usr/bin/clang-20 /usr/bin/clang
152+
ln -sf /usr/bin/clang++-20 /usr/bin/clang++
153+
export CC=/usr/bin/clang
154+
export CXX=/usr/bin/clang++
155+
echo "Compiler versions:"
156+
$CC --version
157+
$CXX --version
158+
159+
# Build
160+
echo "Starting chdb build with Python 3.8..."
161+
pyenv shell 3.8
162+
python --version
163+
echo "Build start time: $(date)"
164+
bash ./chdb/build-musl.sh
165+
echo "Build end time: $(date)"
166+
167+
# Test
168+
echo "Running smoke test with Python 3.9..."
169+
pyenv shell 3.9
170+
python --version
171+
echo "Test start time: $(date)"
172+
bash -x ./chdb/test_smoke.sh
173+
echo "Test end time: $(date)"
174+
175+
# Check build results
176+
echo "Build results summary:"
177+
ccache -s
178+
echo "chdb directory contents:"
179+
ls -lh chdb
180+
echo "Build artifacts size:"
181+
du -sh chdb
182+
183+
# 6. Create and audit wheels
184+
echo "=== Creating and auditing wheels ==="
185+
echo "Wheel creation start time: $(date)"
186+
echo "Available disk space before wheel build: $(df -h .)"
187+
188+
# Build wheels
189+
echo "Building wheels with Python 3.8..."
190+
pyenv shell 3.8
191+
python --version
192+
echo "Running make wheel..."
193+
make wheel
194+
echo "Wheel build completed at: $(date)"
195+
echo "Initial wheel files:"
196+
ls -lh dist/ || echo "No dist directory yet"
197+
198+
# Install patchelf
199+
echo "Installing patchelf for wheel auditing..."
200+
wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
201+
tar -xvf patchelf.tar.gz
202+
cp bin/patchelf /usr/bin/
203+
chmod +x /usr/bin/patchelf
204+
echo "patchelf version: $(patchelf --version)"
205+
206+
# Audit wheels
207+
echo "Auditing wheels with Python 3.13..."
208+
pyenv shell 3.13
209+
python --version
210+
python -m pip install auditwheel
211+
echo "auditwheel version: $(auditwheel --version)"
212+
echo "Starting wheel audit at: $(date)"
213+
auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl
214+
echo "Wheel audit completed at: $(date)"
215+
216+
# Clean up non-musllinux wheels
217+
echo "Cleaning up non-musllinux wheels..."
218+
echo "Before cleanup:"
219+
ls -lh dist/
220+
rm -f dist/*-linux_aarch64.whl
221+
echo "After cleanup:"
222+
ls -lh dist/
223+
echo "Final wheel sizes:"
224+
du -sh dist/*
225+
226+
# 7. Test wheels
227+
echo "=== Testing wheels ==="
228+
echo "Wheel testing start time: $(date)"
229+
echo "Available wheels for testing:"
230+
ls -lh dist/*.whl
231+
echo "Wheel file details:"
232+
file dist/*.whl
233+
234+
TOTAL_TESTS=5
235+
CURRENT_TEST=0
236+
TEST_FAILED=false
237+
238+
for version in 3.9 3.10 3.11 3.12 3.13; do
239+
CURRENT_TEST=$((CURRENT_TEST + 1))
240+
echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ==="
241+
echo "Test start time: $(date)"
242+
243+
echo "Switching to Python $version..."
244+
pyenv shell $version
245+
python --version
246+
echo "pip version: $(python -m pip --version)"
247+
248+
echo "Installing chdb wheel..."
249+
python -m pip install dist/*.whl --force-reinstall
250+
echo "Installation completed at: $(date)"
251+
252+
echo "Running basic query test..."
253+
python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')"
254+
255+
echo "Running full test suite..."
256+
if make test; then
257+
echo "Test suite PASSED for Python $version at: $(date)"
258+
else
259+
echo "Test suite FAILED for Python $version at: $(date)"
260+
TEST_FAILED=true
261+
break
262+
fi
263+
264+
pyenv shell --unset
265+
echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully"
266+
echo ""
267+
done
268+
269+
echo "All wheel tests completed at: $(date)"
270+
271+
# Check if any tests failed
272+
if [ "$TEST_FAILED" = true ]; then
273+
echo "ERROR: One or more test suites failed!"
274+
echo "Test failure detected - aborting build process"
275+
exit 1
276+
fi
277+
278+
# Create test success marker file only if all tests passed
279+
echo "All tests passed successfully!"
280+
echo "Creating test success marker..."
281+
touch /workspace/.test_success_marker
282+
echo "Test success marker created at: $(date)"
283+
284+
# 8. Scan chdb libraries
285+
echo "=== Scanning chdb libraries ==="
286+
FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)"
287+
SQLITE_VULNERABILITIES_FOUND=false
288+
289+
for file in $FILES_TO_SCAN; do
290+
if [ -f "$file" ]; then
291+
echo "=== Scanning $file ==="
292+
SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true)
293+
echo "$SCAN_OUTPUT"
294+
295+
if echo "$SCAN_OUTPUT" | grep -qi sqlite; then
296+
echo "SQLite vulnerability found in $file"
297+
SQLITE_VULNERABILITIES_FOUND=true
298+
fi
299+
fi
300+
done
301+
302+
if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then
303+
echo "SQLite vulnerabilities detected in chdb libraries!"
304+
else
305+
echo "No SQLite vulnerabilities found in chdb libraries"
306+
fi
307+
308+
# Show final results
309+
echo "=== Final wheel files ==="
310+
ls -la ./dist/
311+
continue-on-error: false
312+
# Check test success before upload
313+
- name: Verify test completion
314+
run: |
315+
echo "=== Verifying test completion ==="
316+
if [ ! -f ".test_success_marker" ]; then
317+
echo "ERROR: Test success marker file not found!"
318+
echo "This indicates that the wheel testing did not complete successfully."
319+
echo "Aborting upload process."
320+
exit 1
321+
fi
322+
echo "Test success marker found. All tests completed successfully."
323+
echo "Proceeding with wheel upload..."
324+
continue-on-error: false
325+
# Upload wheels to release
326+
- name: Upload wheels to release
327+
if: startsWith(github.ref, 'refs/tags/v')
328+
run: |
329+
echo "=== Uploading wheels to release ==="
330+
ls -la ./dist/
331+
gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber
332+
env:
333+
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
334+
335+
# Upload to PyPI
336+
- name: Upload to PyPI
337+
if: startsWith(github.ref, 'refs/tags/v')
338+
uses: addnab/docker-run-action@v3
339+
with:
340+
image: quay.io/pypa/musllinux_1_2_aarch64
341+
options: -v ${{ github.workspace }}:/workspace
342+
run: |
343+
cd /workspace
344+
echo "=== Uploading to PyPI ==="
345+
346+
export PATH="$HOME/.pyenv/bin:$PATH"
347+
eval "$(pyenv init -)"
348+
pyenv shell 3.13
349+
python -m twine upload ./dist/*.whl
350+
env:
351+
TWINE_USERNAME: __token__
352+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
353+
354+
# Upload artifacts
355+
- name: Upload build artifacts
356+
if: always()
357+
uses: actions/upload-artifact@v4
358+
with:
359+
name: chdb-artifacts-musllinux-aarch64
360+
path: |
361+
./dist/*.whl
362+
overwrite: true

0 commit comments

Comments
 (0)