From 6679eb5e4da58a20b252a70675d929ae2df16e5e Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 17:34:32 +0800 Subject: [PATCH 01/20] test: use new instances --- .../workflows/build_linux_arm64_wheels-gh.yml | 2 +- chdb/build_pybind11.sh | 25 +++++++++++++++++++ contrib/pybind11-cmake/CMakeLists.txt | 7 ++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 3bfdf772c6d..5ef98c47518 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -23,7 +23,7 @@ on: jobs: build_universal_wheel: name: Build Universal Wheel (Linux ARM64) - runs-on: GH-Linux-ARM64 + runs-on: [self-hosted, linux, arm64, ubuntu-latest] steps: - name: Install Python build dependencies run: | diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh index 33066b916fe..6095f4747e5 100755 --- a/chdb/build_pybind11.sh +++ b/chdb/build_pybind11.sh @@ -33,8 +33,33 @@ build_pybind11_nonlimitedapi() { local py_version=$1 echo "Building pybind11 nonlimitedapi library for Python ${py_version}..." + local full_py_version="" + local custom_python_path="" + + if command -v pyenv >/dev/null 2>&1; then + full_py_version=$(pyenv versions --bare | grep "^${py_version}\." | head -n 1) + + if [ -n "$full_py_version" ]; then + custom_python_path="$HOME/.pyenv/versions/${full_py_version}/bin/python3" + if [ -f "$custom_python_path" ]; then + echo "Found pyenv Python ${full_py_version} at: $custom_python_path" + else + echo "Warning: pyenv Python ${full_py_version} not found at expected path" + custom_python_path="" + fi + else + echo "Warning: Python ${py_version}.x not found in pyenv versions" + fi + else + echo "Warning: pyenv not found" + fi + local py_cmake_args="${CMAKE_ARGS} -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version}" + if [ -n "$custom_python_path" ]; then + py_cmake_args="${py_cmake_args} -DCHDB_CUSTOM_PYTHON_EXECUTABLE=${custom_python_path}" + fi + cmake ${py_cmake_args} -DENABLE_PYTHON=1 .. # Build only the pybind11 targets diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 0427cf47b71..3a34a169049 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -6,6 +6,13 @@ endif() string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") +# Use custom Python path if provided +if (DEFINED CHDB_CUSTOM_PYTHON_EXECUTABLE) + set(Python_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + set(Python3_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + message(STATUS "Using custom Python executable: ${CHDB_CUSTOM_PYTHON_EXECUTABLE}") +endif() + if (DEFINED PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION) find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development) else() From feed3e776c84c81b69615eb88e200b364fb12062 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 17:35:44 +0800 Subject: [PATCH 02/20] chore: update workflow --- .github/workflows/build_macos_x86_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 5f9cf520d80..f58e2230c86 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -133,7 +133,6 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 - token: ${{ secrets.GH_TOKEN }} - name: Update submodules run: | git submodule update --init --recursive --jobs 4 From 6f28d2b54c4135ceb90ef23868e2b5e9b6df4076 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 21:13:58 +0800 Subject: [PATCH 03/20] chore: update workflow --- .github/workflows/build_linux_x86_wheels.yml | 2 +- .../workflows/build_macos_arm64_wheels.yml | 716 ++++++++-------- .github/workflows/build_macos_x86_wheels.yml | 716 ++++++++-------- .../build_musllinux_arm64_wheels.yml | 762 +++++++++--------- .../workflows/build_musllinux_x86_wheels.yml | 578 ++++++------- 5 files changed, 1387 insertions(+), 1387 deletions(-) diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 7c052007a60..0381392fbb7 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -31,7 +31,7 @@ jobs: sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev + libffi-dev liblzma-dev golang-go - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 0c136439d01..41e9b6919dd 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,363 +1,363 @@ -name: Build macOS arm64 +# name: Build macOS arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS ARM64) - runs-on: macos-14-xlarge - steps: - - name: Check machine architecture - run: | - echo "=== Machine Architecture Information ===" - echo "Machine type: $(uname -m)" - echo "Architecture: $(arch)" - echo "System info: $(uname -a)" - echo "Hardware info:" - system_profiler SPHardwareDataType | grep "Chip\|Processor" - if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then - echo "This is an ARM64 (Apple Silicon) machine" - else - echo "This is an x86_64 (Intel) machine" - fi - - name: Free up disk space (Initial) - run: | - # Clean Homebrew cache - brew cleanup -s 2>/dev/null || true - rm -rf "$(brew --cache)" 2>/dev/null || true - sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true - sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true - sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== Disk usage after cleanup ===" - df -h - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS ARM64) +# runs-on: macos-14-xlarge +# steps: +# - name: Check machine architecture +# run: | +# echo "=== Machine Architecture Information ===" +# echo "Machine type: $(uname -m)" +# echo "Architecture: $(arch)" +# echo "System info: $(uname -a)" +# echo "Hardware info:" +# system_profiler SPHardwareDataType | grep "Chip\|Processor" +# if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then +# echo "This is an ARM64 (Apple Silicon) machine" +# else +# echo "This is an x86_64 (Intel) machine" +# fi +# - name: Free up disk space (Initial) +# run: | +# # Clean Homebrew cache +# brew cleanup -s 2>/dev/null || true +# rm -rf "$(brew --cache)" 2>/dev/null || true +# sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true +# sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true +# sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true +# sudo rm -rf /tmp/* 2>/dev/null || true +# echo "=== Disk usage after cleanup ===" +# df -h +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd - brew install openssl@3 || echo "OpenSSL install failed, continuing..." - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which wasm-ld || echo "wasm-ld not found in PATH" - which go - go version - ccache -s | echo "ccache not available yet" - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd +# brew install openssl@3 || echo "OpenSSL install failed, continuing..." +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which wasm-ld || echo "wasm-ld not found in PATH" +# which go +# go version +# ccache -s | echo "ccache not available yet" +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: Free up disk space (Before compilation) - run: | - echo "=== Disk usage before compilation cleanup ===" - df -h - brew cleanup -s 2>/dev/null || true - rm -rf "$(brew --cache)" 2>/dev/null || true - rm -rf ~/.cache/pip 2>/dev/null || true - rm -rf ~/.pyenv/.cache 2>/dev/null || true - rm -rf ~/.cargo/registry/cache 2>/dev/null || true - echo "=== Disk usage after cleanup ===" - df -h - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ runner.os }}-arm64-ccache - max-size: 5G - append-timestamp: true - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Build wheels - run: | - rm -rf chdb/build/ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl - - name: Verify wheel sizes - run: | - echo "=== Wheel sizes ===" - du -sh dist/* - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Free up disk space - run: | - # Clean more build artifacts - rm -rf buildlib/contrib 2>/dev/null || true - rm -rf buildlib/base 2>/dev/null || true - rm -rf buildlib/src 2>/dev/null || true - - name: Test wheel on all Python versions - run: | - ulimit -c unlimited - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall --no-cache-dir - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - python -m pip uninstall -y chdb - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-arm64.tar.gz" - ls -lh core-files-macos-arm64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-arm64 - path: core-files-macos-arm64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-arm64 - path: | - ./dist/*.whl - ./macos-arm64-libchdb.tar.gz - ./macos-arm64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: Free up disk space (Before compilation) +# run: | +# echo "=== Disk usage before compilation cleanup ===" +# df -h +# brew cleanup -s 2>/dev/null || true +# rm -rf "$(brew --cache)" 2>/dev/null || true +# rm -rf ~/.cache/pip 2>/dev/null || true +# rm -rf ~/.pyenv/.cache 2>/dev/null || true +# rm -rf ~/.cargo/registry/cache 2>/dev/null || true +# echo "=== Disk usage after cleanup ===" +# df -h +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ${{ runner.os }}-arm64-ccache +# max-size: 5G +# append-timestamp: true +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Build wheels +# run: | +# rm -rf chdb/build/ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl +# - name: Verify wheel sizes +# run: | +# echo "=== Wheel sizes ===" +# du -sh dist/* +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Free up disk space +# run: | +# # Clean more build artifacts +# rm -rf buildlib/contrib 2>/dev/null || true +# rm -rf buildlib/base 2>/dev/null || true +# rm -rf buildlib/src 2>/dev/null || true +# - name: Test wheel on all Python versions +# run: | +# ulimit -c unlimited +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall --no-cache-dir +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# python -m pip uninstall -y chdb +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-arm64.tar.gz" +# ls -lh core-files-macos-arm64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-arm64 +# path: core-files-macos-arm64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-arm64 +# path: | +# ./dist/*.whl +# ./macos-arm64-libchdb.tar.gz +# ./macos-arm64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index f58e2230c86..8a590dde4ec 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -1,363 +1,363 @@ -name: Build macOS X86 +# name: Build macOS X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS x86_64) - runs-on: macos-15-intel - timeout-minutes: 600 - steps: - - name: Check machine architecture - run: | - echo "=== Machine Architecture Information ===" - echo "Machine type: $(uname -m)" - echo "Architecture: $(arch)" - echo "System info: $(uname -a)" - echo "Hardware info:" - system_profiler SPHardwareDataType | grep "Chip\|Processor" - if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then - echo "This is an ARM64 (Apple Silicon) machine" - else - echo "This is an x86_64 (Intel) machine" - fi - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS x86_64) +# runs-on: macos-15-intel +# timeout-minutes: 600 +# steps: +# - name: Check machine architecture +# run: | +# echo "=== Machine Architecture Information ===" +# echo "Machine type: $(uname -m)" +# echo "Architecture: $(arch)" +# echo "System info: $(uname -a)" +# echo "Hardware info:" +# system_profiler SPHardwareDataType | grep "Chip\|Processor" +# if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then +# echo "This is an ARM64 (Apple Silicon) machine" +# else +# echo "This is an x86_64 (Intel) machine" +# fi +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.9 3.10 3.11 3.12 3.13 3.14 +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.9 3.10 3.11 3.12 3.13 3.14 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which go - go version - ccache -s || echo "ccache not available yet" - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update - # Check SQLite vulnerabilities in Homebrew packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which go +# go version +# ccache -s || echo "ccache not available yet" +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# # Update grype vulnerability database +# grype db update +# # Check SQLite vulnerabilities in Homebrew packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ runner.os }}-x86_64-ccache - max-size: 10G - append-timestamp: true - env: - CCACHE_NOHASHDIR: "true" - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.9 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.9 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.9 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Build wheels - run: | - rm -rf chdb/build/ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.9 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl - - name: Verify wheel sizes - run: | - echo "=== Wheel sizes ===" - du -sh dist/* - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Free up disk space - run: | - # Clean more build artifacts - rm -rf buildlib/contrib 2>/dev/null || true - rm -rf buildlib/base 2>/dev/null || true - rm -rf buildlib/src 2>/dev/null || true - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-x86_64.tar.gz" - ls -lh core-files-macos-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-x86_64 - path: core-files-macos-x86_64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-x86_64 - path: | - ./dist/*.whl - ./macos-x86_64-libchdb.tar.gz - ./macos-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ${{ runner.os }}-x86_64-ccache +# max-size: 10G +# append-timestamp: true +# env: +# CCACHE_NOHASHDIR: "true" +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.9 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.9 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.9 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Build wheels +# run: | +# rm -rf chdb/build/ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.9 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl +# - name: Verify wheel sizes +# run: | +# echo "=== Wheel sizes ===" +# du -sh dist/* +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Free up disk space +# run: | +# # Clean more build artifacts +# rm -rf buildlib/contrib 2>/dev/null || true +# rm -rf buildlib/base 2>/dev/null || true +# rm -rf buildlib/src 2>/dev/null || true +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-x86_64.tar.gz" +# ls -lh core-files-macos-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-x86_64 +# path: core-files-macos-x86_64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-x86_64 +# path: | +# ./dist/*.whl +# ./macos-x86_64-libchdb.tar.gz +# ./macos-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index b76f3ab6f02..57adb7b7abd 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -1,382 +1,382 @@ -name: Build Linux(musllinux) ARM64 - -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' - -jobs: - build_musllinux_wheels: - name: Build musllinux wheels (Alpine Linux aarch64) - runs-on: GH-Linux-ARM64 - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Configure git safe directory - run: | - git config --global --add safe.directory '*' - - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - - name: Build chdb wheels in container - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_2_aarch64 - options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} - run: | - cd /workspace - - # Configure git safe directory in container - apk update - apk add --no-cache git python3 py3-pip py3-setuptools - echo "=== Configure git safe directory ===" - git config --global --add safe.directory /workspace - git describe --tags - python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" - - # 1. Check system info - echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - else - echo "Not musl libc" - fi - echo "Workspace mounted at: /workspace" - ls -la /workspace - - # 2. Install build dependencies - echo "=== Installing build dependencies ===" - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - apk add --no-cache make cmake ccache ninja yasm gawk - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - - # 3. Scan SQLite vulnerabilities - echo "=== Scanning SQLite vulnerabilities ===" - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - grype db update - - # Check SQLite vulnerabilities - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - - # 4. Setup Python environments - echo "=== Setting up Python environments ===" - # Setup pyenv - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - - # Install Python versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing Python $version" - pyenv install $version:latest - done - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - - # Verify installations - echo "Installed versions:" - pyenv versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - - # Install Python dependencies - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - # Update version for release (if triggered by tag) - if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then - pyenv shell 3.9 +# name: Build Linux(musllinux) ARM64 + +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' + +# jobs: +# build_musllinux_wheels: +# name: Build musllinux wheels (Alpine Linux aarch64) +# runs-on: GH-Linux-ARM64 +# steps: +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 + +# - name: Configure git safe directory +# run: | +# git config --global --add safe.directory '*' + +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 + +# - name: Build chdb wheels in container +# uses: addnab/docker-run-action@v3 +# with: +# image: quay.io/pypa/musllinux_1_2_aarch64 +# options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} +# run: | +# cd /workspace + +# # Configure git safe directory in container +# apk update +# apk add --no-cache git python3 py3-pip py3-setuptools +# echo "=== Configure git safe directory ===" +# git config --global --add safe.directory /workspace +# git describe --tags +# python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + +# # 1. Check system info +# echo "=== Container System Info ===" +# echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +# if [ -f /lib/ld-musl-aarch64.so.1 ]; then +# echo "musl libc aarch64" +# elif [ -f /lib/libc.musl-aarch64.so.1 ]; then +# echo "musl libc aarch64" +# else +# echo "Not musl libc" +# fi +# echo "Workspace mounted at: /workspace" +# ls -la /workspace + +# # 2. Install build dependencies +# echo "=== Installing build dependencies ===" +# apk add --no-cache make build-base openssl-dev zlib-dev \ +# bzip2-dev readline-dev sqlite-dev wget curl llvm \ +# ncurses-dev xz-dev tk-dev libxml2-dev \ +# libffi-dev linux-headers +# apk add --no-cache make cmake ccache ninja yasm gawk +# apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + +# # 3. Scan SQLite vulnerabilities +# echo "=== Scanning SQLite vulnerabilities ===" +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# grype db update + +# # Check SQLite vulnerabilities +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" + +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# else +# echo "No SQLite vulnerabilities found" +# fi + +# # 4. Setup Python environments +# echo "=== Setting up Python environments ===" +# # Setup pyenv +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" + +# # Install Python versions +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing Python $version" +# pyenv install $version:latest +# done +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 + +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "Python $version is installed" +# done +# echo "All Python versions verified successfully!" + +# # Install Rust +# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +# source $HOME/.cargo/env +# rustup toolchain install nightly-2025-07-07 +# rustup component add --toolchain nightly-2025-07-07 rust-src + +# # Install Python dependencies +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# if [ "$version" = "3.8" ]; then +# python -m pip install setuptools tox twine psutil wheel +# else +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# fi +# pyenv shell --unset +# done + +# # Update version for release (if triggered by tag) +# if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - fi - - # 5. Build chdb - echo "=== Building chdb ===" - echo "Timestamp: $(date)" - echo "Current directory: $(pwd)" - echo "Available disk space: $(df -h .)" - - # Setup clang - echo "Setting up clang compiler..." - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - echo "Compiler versions:" - $CC --version - $CXX --version - - # Build - echo "Starting chdb build with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Build start time: $(date)" - bash ./chdb/build-musl.sh - echo "Build end time: $(date)" - - # Test - echo "Running smoke test with Python 3.9..." - pyenv shell 3.9 - python --version - echo "Test start time: $(date)" - bash -x ./chdb/test_smoke.sh - echo "Test end time: $(date)" - - # Check build results - echo "Build results summary:" - ccache -s - echo "chdb directory contents:" - ls -lh chdb - echo "Build artifacts size:" - du -sh chdb - - # 6. Create and audit wheels - echo "=== Creating and auditing wheels ===" - echo "Wheel creation start time: $(date)" - echo "Available disk space before wheel build: $(df -h .)" - - # Build wheels - echo "Building wheels with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Running make wheel..." - make wheel - echo "Wheel build completed at: $(date)" - echo "Initial wheel files:" - ls -lh dist/ || echo "No dist directory yet" - - # Install patchelf - echo "Installing patchelf for wheel auditing..." - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - echo "patchelf version: $(patchelf --version)" - - # Audit wheels - echo "Auditing wheels with Python 3.13..." - pyenv shell 3.13 - python --version - python -m pip install auditwheel - echo "auditwheel version: $(auditwheel --version)" - echo "Starting wheel audit at: $(date)" - auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - echo "Wheel audit completed at: $(date)" - - # Clean up non-musllinux wheels - echo "Cleaning up non-musllinux wheels..." - echo "Before cleanup:" - ls -lh dist/ - rm -f dist/*-linux_aarch64.whl - echo "After cleanup:" - ls -lh dist/ - echo "Final wheel sizes:" - du -sh dist/* - - # 7. Test wheels - echo "=== Testing wheels ===" - echo "Wheel testing start time: $(date)" - echo "Available wheels for testing:" - ls -lh dist/*.whl - echo "Wheel file details:" - file dist/*.whl - - TOTAL_TESTS=5 - CURRENT_TEST=0 - TEST_FAILED=false - - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - CURRENT_TEST=$((CURRENT_TEST + 1)) - echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" - echo "Test start time: $(date)" - - echo "Switching to Python $version..." - pyenv shell $version - python --version - echo "pip version: $(python -m pip --version)" - - echo "Installing chdb wheel..." - python -m pip install dist/*.whl --force-reinstall - echo "Installation completed at: $(date)" - - echo "Running basic query test..." - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - - echo "Running full test suite..." - if make test; then - echo "Test suite PASSED for Python $version at: $(date)" - else - echo "Test suite FAILED for Python $version at: $(date)" - TEST_FAILED=true - break - fi - - pyenv shell --unset - echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" - echo "" - done - - echo "All wheel tests completed at: $(date)" - - # Check if any tests failed - if [ "$TEST_FAILED" = true ]; then - echo "ERROR: One or more test suites failed!" - echo "Test failure detected - aborting build process" - exit 1 - fi - - # Create test success marker file only if all tests passed - echo "All tests passed successfully!" - echo "Creating test success marker..." - touch /workspace/.test_success_marker - echo "Test success marker created at: $(date)" - - # 8. Scan chdb libraries - echo "=== Scanning chdb libraries ===" - FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - - # Show final results - echo "=== Final wheel files ===" - ls -la ./dist/ - continue-on-error: false - # Check test success before upload - - name: Verify test completion - run: | - echo "=== Verifying test completion ===" - if [ ! -f ".test_success_marker" ]; then - echo "ERROR: Test success marker file not found!" - echo "This indicates that the wheel testing did not complete successfully." - echo "Aborting upload process." - exit 1 - fi - echo "Test success marker found. All tests completed successfully." - echo "Proceeding with wheel upload..." - continue-on-error: false - # Upload wheels to release - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - echo "=== Uploading wheels to release ===" - ls -la ./dist/ - gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - continue-on-error: true - - - name: Setup Python and install twine for PyPI upload - run: | - echo "=== Setting up Python for PyPI upload ===" - python3 --version - python3 -m pip --version - echo "=== Installing twine ===" - python3 -m pip install --upgrade pip - python3 -m pip install twine - if ! python3 -m twine --version; then - echo "ERROR: Twine installation failed!" - exit 1 - fi - echo "Twine installed successfully" - ls -la ./dist/ - - # Upload to PyPI - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - echo "=== Uploading to PyPI ===" - ls -la ./dist/ - python3 -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - - # Upload artifacts - - name: Upload build artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-musllinux-aarch64 - path: | - ./dist/*.whl - overwrite: true +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# fi + +# # 5. Build chdb +# echo "=== Building chdb ===" +# echo "Timestamp: $(date)" +# echo "Current directory: $(pwd)" +# echo "Available disk space: $(df -h .)" + +# # Setup clang +# echo "Setting up clang compiler..." +# ln -sf /usr/bin/clang-20 /usr/bin/clang +# ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# echo "Compiler versions:" +# $CC --version +# $CXX --version + +# # Build +# echo "Starting chdb build with Python 3.8..." +# pyenv shell 3.8 +# python --version +# echo "Build start time: $(date)" +# bash ./chdb/build-musl.sh +# echo "Build end time: $(date)" + +# # Test +# echo "Running smoke test with Python 3.9..." +# pyenv shell 3.9 +# python --version +# echo "Test start time: $(date)" +# bash -x ./chdb/test_smoke.sh +# echo "Test end time: $(date)" + +# # Check build results +# echo "Build results summary:" +# ccache -s +# echo "chdb directory contents:" +# ls -lh chdb +# echo "Build artifacts size:" +# du -sh chdb + +# # 6. Create and audit wheels +# echo "=== Creating and auditing wheels ===" +# echo "Wheel creation start time: $(date)" +# echo "Available disk space before wheel build: $(df -h .)" + +# # Build wheels +# echo "Building wheels with Python 3.8..." +# pyenv shell 3.8 +# python --version +# echo "Running make wheel..." +# make wheel +# echo "Wheel build completed at: $(date)" +# echo "Initial wheel files:" +# ls -lh dist/ || echo "No dist directory yet" + +# # Install patchelf +# echo "Installing patchelf for wheel auditing..." +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# cp bin/patchelf /usr/bin/ +# chmod +x /usr/bin/patchelf +# echo "patchelf version: $(patchelf --version)" + +# # Audit wheels +# echo "Auditing wheels with Python 3.13..." +# pyenv shell 3.13 +# python --version +# python -m pip install auditwheel +# echo "auditwheel version: $(auditwheel --version)" +# echo "Starting wheel audit at: $(date)" +# auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl +# echo "Wheel audit completed at: $(date)" + +# # Clean up non-musllinux wheels +# echo "Cleaning up non-musllinux wheels..." +# echo "Before cleanup:" +# ls -lh dist/ +# rm -f dist/*-linux_aarch64.whl +# echo "After cleanup:" +# ls -lh dist/ +# echo "Final wheel sizes:" +# du -sh dist/* + +# # 7. Test wheels +# echo "=== Testing wheels ===" +# echo "Wheel testing start time: $(date)" +# echo "Available wheels for testing:" +# ls -lh dist/*.whl +# echo "Wheel file details:" +# file dist/*.whl + +# TOTAL_TESTS=5 +# CURRENT_TEST=0 +# TEST_FAILED=false + +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# CURRENT_TEST=$((CURRENT_TEST + 1)) +# echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" +# echo "Test start time: $(date)" + +# echo "Switching to Python $version..." +# pyenv shell $version +# python --version +# echo "pip version: $(python -m pip --version)" + +# echo "Installing chdb wheel..." +# python -m pip install dist/*.whl --force-reinstall +# echo "Installation completed at: $(date)" + +# echo "Running basic query test..." +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + +# echo "Running full test suite..." +# if make test; then +# echo "Test suite PASSED for Python $version at: $(date)" +# else +# echo "Test suite FAILED for Python $version at: $(date)" +# TEST_FAILED=true +# break +# fi + +# pyenv shell --unset +# echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" +# echo "" +# done + +# echo "All wheel tests completed at: $(date)" + +# # Check if any tests failed +# if [ "$TEST_FAILED" = true ]; then +# echo "ERROR: One or more test suites failed!" +# echo "Test failure detected - aborting build process" +# exit 1 +# fi + +# # Create test success marker file only if all tests passed +# echo "All tests passed successfully!" +# echo "Creating test success marker..." +# touch /workspace/.test_success_marker +# echo "Test success marker created at: $(date)" + +# # 8. Scan chdb libraries +# echo "=== Scanning chdb libraries ===" +# FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false + +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" + +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done + +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "SQLite vulnerabilities detected in chdb libraries!" +# else +# echo "No SQLite vulnerabilities found in chdb libraries" +# fi + +# # Show final results +# echo "=== Final wheel files ===" +# ls -la ./dist/ +# continue-on-error: false +# # Check test success before upload +# - name: Verify test completion +# run: | +# echo "=== Verifying test completion ===" +# if [ ! -f ".test_success_marker" ]; then +# echo "ERROR: Test success marker file not found!" +# echo "This indicates that the wheel testing did not complete successfully." +# echo "Aborting upload process." +# exit 1 +# fi +# echo "Test success marker found. All tests completed successfully." +# echo "Proceeding with wheel upload..." +# continue-on-error: false +# # Upload wheels to release +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# echo "=== Uploading wheels to release ===" +# ls -la ./dist/ +# gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# continue-on-error: true + +# - name: Setup Python and install twine for PyPI upload +# run: | +# echo "=== Setting up Python for PyPI upload ===" +# python3 --version +# python3 -m pip --version +# echo "=== Installing twine ===" +# python3 -m pip install --upgrade pip +# python3 -m pip install twine +# if ! python3 -m twine --version; then +# echo "ERROR: Twine installation failed!" +# exit 1 +# fi +# echo "Twine installed successfully" +# ls -la ./dist/ + +# # Upload to PyPI +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# echo "=== Uploading to PyPI ===" +# ls -la ./dist/ +# python3 -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + +# # Upload artifacts +# - name: Upload build artifacts +# if: always() +# uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-musllinux-aarch64 +# path: | +# ./dist/*.whl +# overwrite: true diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index e6379a9b27e..79bb3c2c47d 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -1,304 +1,304 @@ -name: Build Linux(musllinux) x86 +# name: Build Linux(musllinux) x86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_musllinux_wheels: - name: Build musllinux wheels (Alpine Linux x86_64) - runs-on: gh-64c - container: - image: quay.io/pypa/musllinux_1_2_x86_64 - options: --privileged - steps: - - name: Check system info - run: | - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-x86_64.so.1 ]; then - echo "musl libc x86_64" - elif [ -f /lib/libc.musl-x86_64.so.1 ]; then - echo "musl libc x86_64" - else - echo "Not musl libc" - fi +# jobs: +# build_musllinux_wheels: +# name: Build musllinux wheels (Alpine Linux x86_64) +# runs-on: gh-64c +# container: +# image: quay.io/pypa/musllinux_1_2_x86_64 +# options: --privileged +# steps: +# - name: Check system info +# run: | +# echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +# if [ -f /lib/ld-musl-x86_64.so.1 ]; then +# echo "musl libc x86_64" +# elif [ -f /lib/libc.musl-x86_64.so.1 ]; then +# echo "musl libc x86_64" +# else +# echo "Not musl libc" +# fi - echo "=== CPU Information ===" - cat /proc/cpuinfo - echo "" - echo "=== Checking CPU requirements ===" - if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then - echo "CPU meets minimum requirements" - else - echo "CPU does not meet minimum requirements" - fi - - name: Install Python build dependencies - run: | - apk update - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# echo "=== CPU Information ===" +# cat /proc/cpuinfo +# echo "" +# echo "=== Checking CPU requirements ===" +# if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then +# echo "CPU meets minimum requirements" +# else +# echo "CPU does not meet minimum requirements" +# fi +# - name: Install Python build dependencies +# run: | +# apk update +# apk add --no-cache make build-base openssl-dev zlib-dev \ +# bzip2-dev readline-dev sqlite-dev wget curl llvm \ +# ncurses-dev xz-dev tk-dev libxml2-dev \ +# libffi-dev linux-headers +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# else +# echo "No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - name: Install clang++ for Alpine - run: | - apk add --no-cache make cmake ccache ninja yasm gawk wget - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - # Install Rust toolchain via rustup for proper target management - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - rustc --version - cargo --version - ccache -s - - name: Update git - run: | - apk add --no-cache git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Configure git safe directory - run: | - git config --global --add safe.directory '*' - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# if [ "$version" = "3.8" ]; then +# python -m pip install setuptools tox twine psutil wheel +# else +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# fi +# pyenv shell --unset +# done +# - name: Install clang++ for Alpine +# run: | +# apk add --no-cache make cmake ccache ninja yasm gawk wget +# apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 +# # Install Rust toolchain via rustup for proper target management +# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +# source $HOME/.cargo/env +# rustup toolchain install nightly-2025-07-07 +# rustup component add --toolchain nightly-2025-07-07 rust-src +# rustc --version +# cargo --version +# ccache -s +# - name: Update git +# run: | +# apk add --no-cache git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Configure git safe directory +# run: | +# git config --global --add safe.directory '*' +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: musllinux-1-2-x86_64 - max-size: 5G - append-timestamp: true - - name: setup clang and link clang-20 to clang - run: | - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build-musl.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source $HOME/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build-musl.sh - pyenv shell 3.9 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: musllinux-1-2-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: setup clang and link clang-20 to clang +# run: | +# ln -sf /usr/bin/clang-20 /usr/bin/clang +# ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build-musl.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source $HOME/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build-musl.sh +# pyenv shell 3.9 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m pip install auditwheel - auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Install GitHub CLI - run: | - # Install GitHub CLI from binary release (Alpine Linux compatible) - wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_amd64.tar.gz -O gh.tar.gz - tar -xf gh.tar.gz - cp gh_*/bin/gh /usr/local/bin/ - chmod +x /usr/local/bin/gh - if ! gh --version; then - echo "ERROR: GitHub CLI installation failed!" - exit 1 - fi - echo "GitHub CLI installed successfully" - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-musllinux-x86_64 - path: | - ./dist/*.whl - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "SQLite vulnerabilities detected in chdb libraries!" +# else +# echo "No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# cp bin/patchelf /usr/bin/ +# chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.13 +# python -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Install GitHub CLI +# run: | +# # Install GitHub CLI from binary release (Alpine Linux compatible) +# wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_amd64.tar.gz -O gh.tar.gz +# tar -xf gh.tar.gz +# cp gh_*/bin/gh /usr/local/bin/ +# chmod +x /usr/local/bin/gh +# if ! gh --version; then +# echo "ERROR: GitHub CLI installation failed!" +# exit 1 +# fi +# echo "GitHub CLI installed successfully" +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-musllinux-x86_64 +# path: | +# ./dist/*.whl +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.13 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done From d4648fb03fcb7bc38bf7c577c53b889839bf4c83 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 21:14:56 +0800 Subject: [PATCH 04/20] chore: update workflow --- .github/workflows/build_linux_arm64_wheels-gh.yml | 2 +- .github/workflows/build_linux_x86_wheels.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 5ef98c47518..178ef1472af 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -31,7 +31,7 @@ jobs: sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev + libffi-dev liblzma-dev golang-go - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 0381392fbb7..7c052007a60 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -31,7 +31,7 @@ jobs: sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev golang-go + libffi-dev liblzma-dev - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools From ad4c47077f95484f7ba75195630476b535564a68 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 21:16:58 +0800 Subject: [PATCH 05/20] chore: update workflow --- .github/workflows/build_linux_x86_wheels.yml | 698 +++++++++---------- 1 file changed, 349 insertions(+), 349 deletions(-) diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 7c052007a60..f3aacb2151b 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -1,364 +1,364 @@ -name: Build Linux X86 +# name: Build Linux X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux x86_64) - runs-on: gh-64c - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux x86_64) +# runs-on: gh-64c +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: true - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: true +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel jupyter nbconvert - pyenv shell --unset - done - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - # Install WebAssembly linker (wasm-ld) - sudo apt-get install -y lld-19 - # Create symlink for wasm-ld - if ! command -v wasm-ld &> /dev/null; then - sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true - fi - which wasm-ld || echo "wasm-ld not found in PATH" - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel jupyter nbconvert +# pyenv shell --unset +# done +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# # Install WebAssembly linker (wasm-ld) +# sudo apt-get install -y lld-19 +# # Create symlink for wasm-ld +# if ! command -v wasm-ld &> /dev/null; then +# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true +# fi +# which wasm-ld || echo "wasm-ld not found in PATH" +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-22.04-x86_64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-22.04-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - sudo rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-x86_64.tar.gz" - ls -lh core-files-linux-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-x86_64 - path: core-files-linux-x86_64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-x86_64 - path: | - ./dist/*.whl - ./linux-x86_64-libchdb.tar.gz - ./linux-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# sudo rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-x86_64.tar.gz" +# ls -lh core-files-linux-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-x86_64 +# path: core-files-linux-x86_64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-x86_64 +# path: | +# ./dist/*.whl +# ./linux-x86_64-libchdb.tar.gz +# ./linux-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} From 6cf187cc45ce53272b5ec97e4c4a6d35a07399b1 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 21:20:54 +0800 Subject: [PATCH 06/20] chore: update workflow --- .github/workflows/build_linux_arm64_wheels-gh.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 178ef1472af..ac51c75afa9 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -35,7 +35,9 @@ jobs: - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + mkdir -p $HOME/.local/bin + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH sudo apt-get update && sudo apt-get install -y jq lsb-release # Detect OS distribution info From 6c7e210cb4bc784085811989d00a1d0f5a389e87 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Mon, 17 Nov 2025 23:19:00 +0800 Subject: [PATCH 07/20] chore: update workflow --- .github/workflows/build_linux_arm64_wheels-gh.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index ac51c75afa9..c2e55632132 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -46,11 +46,11 @@ jobs: echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" # Update grype vulnerability database - grype db update + $HOME/.local/bin/grype db update # Check SQLite vulnerabilities in installed packages echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) + GRYPE_RAW_OUTPUT=$($HOME/.local/bin/grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) echo "Raw grype output:" echo "$GRYPE_RAW_OUTPUT" @@ -213,7 +213,7 @@ jobs: for file in $FILES_TO_SCAN; do if [ -f "$file" ]; then echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + SCAN_OUTPUT=$($HOME/.local/bin/grype "$file" 2>/dev/null || true) echo "$SCAN_OUTPUT" if echo "$SCAN_OUTPUT" | grep -qi sqlite; then From b0a1f0e660db2ff26914fc59a47665889c950327 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Wed, 19 Nov 2025 18:04:04 +0800 Subject: [PATCH 08/20] chore: update workflow --- .github/workflows/build_linux_arm64_wheels-gh.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index c2e55632132..1716d6cf399 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -256,7 +256,10 @@ jobs: patchelf --version - name: Audit wheels run: | - python3 -m pip install auditwheel + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.9 + python -m pip install auditwheel auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl continue-on-error: false - name: Show files From 61a32aa0b8cde6572f09ec70a7237193647f7ccd Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 02:18:32 +0800 Subject: [PATCH 09/20] chore: update workflow --- .../build_musllinux_arm64_wheels.yml | 762 +++++++++--------- 1 file changed, 381 insertions(+), 381 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 57adb7b7abd..f833cd388de 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -1,382 +1,382 @@ -# name: Build Linux(musllinux) ARM64 - -# on: -# workflow_dispatch: -# inputs: -# TAG_NAME: -# description: 'Release Version Tag' -# required: true -# release: -# types: [created] -# push: -# branches: -# - main -# paths-ignore: -# - '**/*.md' -# pull_request: -# branches: -# - main -# paths-ignore: -# - '**/*.md' - -# jobs: -# build_musllinux_wheels: -# name: Build musllinux wheels (Alpine Linux aarch64) -# runs-on: GH-Linux-ARM64 -# steps: -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 - -# - name: Configure git safe directory -# run: | -# git config --global --add safe.directory '*' - -# - name: Update submodules -# run: | -# git submodule update --init --recursive --jobs 4 - -# - name: Build chdb wheels in container -# uses: addnab/docker-run-action@v3 -# with: -# image: quay.io/pypa/musllinux_1_2_aarch64 -# options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} -# run: | -# cd /workspace - -# # Configure git safe directory in container -# apk update -# apk add --no-cache git python3 py3-pip py3-setuptools -# echo "=== Configure git safe directory ===" -# git config --global --add safe.directory /workspace -# git describe --tags -# python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" - -# # 1. Check system info -# echo "=== Container System Info ===" -# echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" -# if [ -f /lib/ld-musl-aarch64.so.1 ]; then -# echo "musl libc aarch64" -# elif [ -f /lib/libc.musl-aarch64.so.1 ]; then -# echo "musl libc aarch64" -# else -# echo "Not musl libc" -# fi -# echo "Workspace mounted at: /workspace" -# ls -la /workspace - -# # 2. Install build dependencies -# echo "=== Installing build dependencies ===" -# apk add --no-cache make build-base openssl-dev zlib-dev \ -# bzip2-dev readline-dev sqlite-dev wget curl llvm \ -# ncurses-dev xz-dev tk-dev libxml2-dev \ -# libffi-dev linux-headers -# apk add --no-cache make cmake ccache ninja yasm gawk -# apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - -# # 3. Scan SQLite vulnerabilities -# echo "=== Scanning SQLite vulnerabilities ===" -# # Install grype -# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin -# grype db update - -# # Check SQLite vulnerabilities -# echo "Scanning SQLite packages for vulnerabilities..." -# GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) -# echo "Raw grype output:" -# echo "$GRYPE_RAW_OUTPUT" - -# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) -# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then -# echo "SQLite vulnerabilities found in packages! Build should be reviewed." -# echo "SQLite vulnerability details:" -# echo "$SQLITE_SCAN_OUTPUT" -# else -# echo "No SQLite vulnerabilities found" -# fi - -# # 4. Setup Python environments -# echo "=== Setting up Python environments ===" -# # Setup pyenv -# curl https://pyenv.run | bash -# export PATH="$HOME/.pyenv/bin:$PATH" -# eval "$(pyenv init -)" - -# # Install Python versions -# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do -# echo "Installing Python $version" -# pyenv install $version:latest -# done -# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - -# # Verify installations -# echo "Installed versions:" -# pyenv versions -# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do -# if ! pyenv versions --bare | grep -q "^$version"; then -# echo "ERROR: Python $version is not installed!" -# exit 1 -# fi -# echo "Python $version is installed" -# done -# echo "All Python versions verified successfully!" - -# # Install Rust -# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable -# source $HOME/.cargo/env -# rustup toolchain install nightly-2025-07-07 -# rustup component add --toolchain nightly-2025-07-07 rust-src - -# # Install Python dependencies -# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do -# echo "Installing dependencies for Python $version" -# pyenv shell $version -# python -m pip install --upgrade pip -# if [ "$version" = "3.8" ]; then -# python -m pip install setuptools tox twine psutil wheel -# else -# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel -# fi -# pyenv shell --unset -# done - -# # Update version for release (if triggered by tag) -# if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then -# pyenv shell 3.9 +name: Build Linux(musllinux) ARM64 + +on: + workflow_dispatch: + inputs: + TAG_NAME: + description: 'Release Version Tag' + required: true + release: + types: [created] + push: + branches: + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - main + paths-ignore: + - '**/*.md' + +jobs: + build_musllinux_wheels: + name: Build musllinux wheels (Alpine Linux aarch64) + runs-on: [self-hosted, linux, arm64, ubuntu-latest] + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Configure git safe directory + run: | + git config --global --add safe.directory '*' + + - name: Update submodules + run: | + git submodule update --init --recursive --jobs 4 + + - name: Build chdb wheels in container + uses: addnab/docker-run-action@v3 + with: + image: quay.io/pypa/musllinux_1_2_aarch64 + options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} + run: | + cd /workspace + + # Configure git safe directory in container + apk update + apk add --no-cache git python3 py3-pip py3-setuptools + echo "=== Configure git safe directory ===" + git config --global --add safe.directory /workspace + git describe --tags + python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + + # 1. Check system info + echo "=== Container System Info ===" + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" + else + echo "Not musl libc" + fi + echo "Workspace mounted at: /workspace" + ls -la /workspace + + # 2. Install build dependencies + echo "=== Installing build dependencies ===" + apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers + apk add --no-cache make cmake ccache ninja yasm gawk + apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + + # 3. Scan SQLite vulnerabilities + echo "=== Scanning SQLite vulnerabilities ===" + # Install grype + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + grype db update + + # Check SQLite vulnerabilities + echo "Scanning SQLite packages for vulnerabilities..." + GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) + echo "Raw grype output:" + echo "$GRYPE_RAW_OUTPUT" + + SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) + if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" + else + echo "No SQLite vulnerabilities found" + fi + + # 4. Setup Python environments + echo "=== Setting up Python environments ===" + # Setup pyenv + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + + # Install Python versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing Python $version" + pyenv install $version:latest + done + pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 + + # Verify installations + echo "Installed versions:" + pyenv versions + for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" + done + echo "All Python versions verified successfully!" + + # Install Rust + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable + source $HOME/.cargo/env + rustup toolchain install nightly-2025-07-07 + rustup component add --toolchain nightly-2025-07-07 rust-src + + # Install Python dependencies + for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset + done + + # Update version for release (if triggered by tag) + if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then + pyenv shell 3.9 -# # Install bump-my-version -# python -m pip install bump-my-version -# TAG_NAME=${GITHUB_REF#refs/tags/v} -# bump-my-version replace --new-version $TAG_NAME -# echo "Version files updated to $TAG_NAME" -# pyenv shell --unset -# fi - -# # 5. Build chdb -# echo "=== Building chdb ===" -# echo "Timestamp: $(date)" -# echo "Current directory: $(pwd)" -# echo "Available disk space: $(df -h .)" - -# # Setup clang -# echo "Setting up clang compiler..." -# ln -sf /usr/bin/clang-20 /usr/bin/clang -# ln -sf /usr/bin/clang++-20 /usr/bin/clang++ -# export CC=/usr/bin/clang -# export CXX=/usr/bin/clang++ -# echo "Compiler versions:" -# $CC --version -# $CXX --version - -# # Build -# echo "Starting chdb build with Python 3.8..." -# pyenv shell 3.8 -# python --version -# echo "Build start time: $(date)" -# bash ./chdb/build-musl.sh -# echo "Build end time: $(date)" - -# # Test -# echo "Running smoke test with Python 3.9..." -# pyenv shell 3.9 -# python --version -# echo "Test start time: $(date)" -# bash -x ./chdb/test_smoke.sh -# echo "Test end time: $(date)" - -# # Check build results -# echo "Build results summary:" -# ccache -s -# echo "chdb directory contents:" -# ls -lh chdb -# echo "Build artifacts size:" -# du -sh chdb - -# # 6. Create and audit wheels -# echo "=== Creating and auditing wheels ===" -# echo "Wheel creation start time: $(date)" -# echo "Available disk space before wheel build: $(df -h .)" - -# # Build wheels -# echo "Building wheels with Python 3.8..." -# pyenv shell 3.8 -# python --version -# echo "Running make wheel..." -# make wheel -# echo "Wheel build completed at: $(date)" -# echo "Initial wheel files:" -# ls -lh dist/ || echo "No dist directory yet" - -# # Install patchelf -# echo "Installing patchelf for wheel auditing..." -# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz -# tar -xvf patchelf.tar.gz -# cp bin/patchelf /usr/bin/ -# chmod +x /usr/bin/patchelf -# echo "patchelf version: $(patchelf --version)" - -# # Audit wheels -# echo "Auditing wheels with Python 3.13..." -# pyenv shell 3.13 -# python --version -# python -m pip install auditwheel -# echo "auditwheel version: $(auditwheel --version)" -# echo "Starting wheel audit at: $(date)" -# auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl -# echo "Wheel audit completed at: $(date)" - -# # Clean up non-musllinux wheels -# echo "Cleaning up non-musllinux wheels..." -# echo "Before cleanup:" -# ls -lh dist/ -# rm -f dist/*-linux_aarch64.whl -# echo "After cleanup:" -# ls -lh dist/ -# echo "Final wheel sizes:" -# du -sh dist/* - -# # 7. Test wheels -# echo "=== Testing wheels ===" -# echo "Wheel testing start time: $(date)" -# echo "Available wheels for testing:" -# ls -lh dist/*.whl -# echo "Wheel file details:" -# file dist/*.whl - -# TOTAL_TESTS=5 -# CURRENT_TEST=0 -# TEST_FAILED=false - -# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do -# CURRENT_TEST=$((CURRENT_TEST + 1)) -# echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" -# echo "Test start time: $(date)" - -# echo "Switching to Python $version..." -# pyenv shell $version -# python --version -# echo "pip version: $(python -m pip --version)" - -# echo "Installing chdb wheel..." -# python -m pip install dist/*.whl --force-reinstall -# echo "Installation completed at: $(date)" - -# echo "Running basic query test..." -# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - -# echo "Running full test suite..." -# if make test; then -# echo "Test suite PASSED for Python $version at: $(date)" -# else -# echo "Test suite FAILED for Python $version at: $(date)" -# TEST_FAILED=true -# break -# fi - -# pyenv shell --unset -# echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" -# echo "" -# done - -# echo "All wheel tests completed at: $(date)" - -# # Check if any tests failed -# if [ "$TEST_FAILED" = true ]; then -# echo "ERROR: One or more test suites failed!" -# echo "Test failure detected - aborting build process" -# exit 1 -# fi - -# # Create test success marker file only if all tests passed -# echo "All tests passed successfully!" -# echo "Creating test success marker..." -# touch /workspace/.test_success_marker -# echo "Test success marker created at: $(date)" - -# # 8. Scan chdb libraries -# echo "=== Scanning chdb libraries ===" -# FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" -# SQLITE_VULNERABILITIES_FOUND=false - -# for file in $FILES_TO_SCAN; do -# if [ -f "$file" ]; then -# echo "=== Scanning $file ===" -# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) -# echo "$SCAN_OUTPUT" - -# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then -# echo "SQLite vulnerability found in $file" -# SQLITE_VULNERABILITIES_FOUND=true -# fi -# fi -# done - -# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then -# echo "SQLite vulnerabilities detected in chdb libraries!" -# else -# echo "No SQLite vulnerabilities found in chdb libraries" -# fi - -# # Show final results -# echo "=== Final wheel files ===" -# ls -la ./dist/ -# continue-on-error: false -# # Check test success before upload -# - name: Verify test completion -# run: | -# echo "=== Verifying test completion ===" -# if [ ! -f ".test_success_marker" ]; then -# echo "ERROR: Test success marker file not found!" -# echo "This indicates that the wheel testing did not complete successfully." -# echo "Aborting upload process." -# exit 1 -# fi -# echo "Test success marker found. All tests completed successfully." -# echo "Proceeding with wheel upload..." -# continue-on-error: false -# # Upload wheels to release -# - name: Upload wheels to release -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# echo "=== Uploading wheels to release ===" -# ls -la ./dist/ -# gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber -# env: -# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} -# continue-on-error: true - -# - name: Setup Python and install twine for PyPI upload -# run: | -# echo "=== Setting up Python for PyPI upload ===" -# python3 --version -# python3 -m pip --version -# echo "=== Installing twine ===" -# python3 -m pip install --upgrade pip -# python3 -m pip install twine -# if ! python3 -m twine --version; then -# echo "ERROR: Twine installation failed!" -# exit 1 -# fi -# echo "Twine installed successfully" -# ls -la ./dist/ - -# # Upload to PyPI -# - name: Upload pypi -# if: startsWith(github.ref, 'refs/tags/v') -# run: | -# echo "=== Uploading to PyPI ===" -# ls -la ./dist/ -# python3 -m twine upload dist/*.whl -# env: -# TWINE_USERNAME: __token__ -# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - -# # Upload artifacts -# - name: Upload build artifacts -# if: always() -# uses: actions/upload-artifact@v4 -# with: -# name: chdb-artifacts-musllinux-aarch64 -# path: | -# ./dist/*.whl -# overwrite: true + # Install bump-my-version + python -m pip install bump-my-version + TAG_NAME=${GITHUB_REF#refs/tags/v} + bump-my-version replace --new-version $TAG_NAME + echo "Version files updated to $TAG_NAME" + pyenv shell --unset + fi + + # 5. Build chdb + echo "=== Building chdb ===" + echo "Timestamp: $(date)" + echo "Current directory: $(pwd)" + echo "Available disk space: $(df -h .)" + + # Setup clang + echo "Setting up clang compiler..." + ln -sf /usr/bin/clang-20 /usr/bin/clang + ln -sf /usr/bin/clang++-20 /usr/bin/clang++ + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + echo "Compiler versions:" + $CC --version + $CXX --version + + # Build + echo "Starting chdb build with Python 3.8..." + pyenv shell 3.8 + python --version + echo "Build start time: $(date)" + bash ./chdb/build-musl.sh + echo "Build end time: $(date)" + + # Test + echo "Running smoke test with Python 3.9..." + pyenv shell 3.9 + python --version + echo "Test start time: $(date)" + bash -x ./chdb/test_smoke.sh + echo "Test end time: $(date)" + + # Check build results + echo "Build results summary:" + ccache -s + echo "chdb directory contents:" + ls -lh chdb + echo "Build artifacts size:" + du -sh chdb + + # 6. Create and audit wheels + echo "=== Creating and auditing wheels ===" + echo "Wheel creation start time: $(date)" + echo "Available disk space before wheel build: $(df -h .)" + + # Build wheels + echo "Building wheels with Python 3.8..." + pyenv shell 3.8 + python --version + echo "Running make wheel..." + make wheel + echo "Wheel build completed at: $(date)" + echo "Initial wheel files:" + ls -lh dist/ || echo "No dist directory yet" + + # Install patchelf + echo "Installing patchelf for wheel auditing..." + wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz + tar -xvf patchelf.tar.gz + cp bin/patchelf /usr/bin/ + chmod +x /usr/bin/patchelf + echo "patchelf version: $(patchelf --version)" + + # Audit wheels + echo "Auditing wheels with Python 3.13..." + pyenv shell 3.13 + python --version + python -m pip install auditwheel + echo "auditwheel version: $(auditwheel --version)" + echo "Starting wheel audit at: $(date)" + auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl + echo "Wheel audit completed at: $(date)" + + # Clean up non-musllinux wheels + echo "Cleaning up non-musllinux wheels..." + echo "Before cleanup:" + ls -lh dist/ + rm -f dist/*-linux_aarch64.whl + echo "After cleanup:" + ls -lh dist/ + echo "Final wheel sizes:" + du -sh dist/* + + # 7. Test wheels + echo "=== Testing wheels ===" + echo "Wheel testing start time: $(date)" + echo "Available wheels for testing:" + ls -lh dist/*.whl + echo "Wheel file details:" + file dist/*.whl + + TOTAL_TESTS=5 + CURRENT_TEST=0 + TEST_FAILED=false + + for version in 3.9 3.10 3.11 3.12 3.13 3.14; do + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." + pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." + python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Running basic query test..." + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi + + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" + done + + echo "All wheel tests completed at: $(date)" + + # Check if any tests failed + if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 + fi + + # Create test success marker file only if all tests passed + echo "All tests passed successfully!" + echo "Creating test success marker..." + touch /workspace/.test_success_marker + echo "Test success marker created at: $(date)" + + # 8. Scan chdb libraries + echo "=== Scanning chdb libraries ===" + FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" + SQLITE_VULNERABILITIES_FOUND=false + + for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi + done + + if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" + else + echo "No SQLite vulnerabilities found in chdb libraries" + fi + + # Show final results + echo "=== Final wheel files ===" + ls -la ./dist/ + continue-on-error: false + # Check test success before upload + - name: Verify test completion + run: | + echo "=== Verifying test completion ===" + if [ ! -f ".test_success_marker" ]; then + echo "ERROR: Test success marker file not found!" + echo "This indicates that the wheel testing did not complete successfully." + echo "Aborting upload process." + exit 1 + fi + echo "Test success marker found. All tests completed successfully." + echo "Proceeding with wheel upload..." + continue-on-error: false + # Upload wheels to release + - name: Upload wheels to release + if: startsWith(github.ref, 'refs/tags/v') + run: | + echo "=== Uploading wheels to release ===" + ls -la ./dist/ + gh release upload ${{ github.ref_name }} ./dist/*.whl --clobber + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + continue-on-error: true + + - name: Setup Python and install twine for PyPI upload + run: | + echo "=== Setting up Python for PyPI upload ===" + python3 --version + python3 -m pip --version + echo "=== Installing twine ===" + python3 -m pip install --upgrade pip + python3 -m pip install twine + if ! python3 -m twine --version; then + echo "ERROR: Twine installation failed!" + exit 1 + fi + echo "Twine installed successfully" + ls -la ./dist/ + + # Upload to PyPI + - name: Upload pypi + if: startsWith(github.ref, 'refs/tags/v') + run: | + echo "=== Uploading to PyPI ===" + ls -la ./dist/ + python3 -m twine upload dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + + # Upload artifacts + - name: Upload build artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: chdb-artifacts-musllinux-aarch64 + path: | + ./dist/*.whl + overwrite: true From f1ed3ce2abe92a72af0e753306f797b56314351d Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 02:20:31 +0800 Subject: [PATCH 10/20] chore: update workflow --- .github/workflows/build_musllinux_arm64_wheels.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index f833cd388de..06f78056cfc 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -348,6 +348,9 @@ jobs: - name: Setup Python and install twine for PyPI upload run: | echo "=== Setting up Python for PyPI upload ===" + # Install pip if not available + sudo apt-get update + sudo apt-get install -y python3-pip python3 --version python3 -m pip --version echo "=== Installing twine ===" From 5d434a64f119be921407b597d14730a572b98381 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 14:18:25 +0800 Subject: [PATCH 11/20] fix: fix docker --- .../workflows/build_musllinux_arm64_wheels.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 06f78056cfc..e91b951776e 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -24,6 +24,22 @@ jobs: name: Build musllinux wheels (Alpine Linux aarch64) runs-on: [self-hosted, linux, arm64, ubuntu-latest] steps: + - name: Setup Docker permissions + run: | + # Ensure Docker is running + sudo systemctl start docker + sudo systemctl enable docker + + # Add current user to docker group + sudo usermod -aG docker $USER + + # Set proper permissions on docker socket + sudo chmod 666 /var/run/docker.sock + + # Verify Docker is working + docker --version + docker info + - uses: actions/checkout@v3 with: fetch-depth: 0 From 17690dfd079ec03153f4442e04b50ee31d54e9fb Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 17:10:34 +0800 Subject: [PATCH 12/20] fix: fix docker --- .github/workflows/build_musllinux_arm64_wheels.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index e91b951776e..a3c4d5641f1 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -26,6 +26,10 @@ jobs: steps: - name: Setup Docker permissions run: | + # Update Docker to latest version + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + # Ensure Docker is running sudo systemctl start docker sudo systemctl enable docker From 13ad319ffe24aa8776a91bc82624fb4441e5ae9b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 17:14:51 +0800 Subject: [PATCH 13/20] fix: fix docker --- .../build_musllinux_arm64_wheels.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index a3c4d5641f1..748cb85b0c2 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -26,9 +26,23 @@ jobs: steps: - name: Setup Docker permissions run: | - # Update Docker to latest version - curl -fsSL https://get.docker.com -o get-docker.sh - sudo sh get-docker.sh + # Remove old Docker installation + sudo apt-get remove -y docker docker-engine docker.io containerd runc || true + + # Install Docker from official repository + sudo apt-get update + sudo apt-get install -y ca-certificates curl gnupg lsb-release + + # Add Docker's official GPG key + sudo mkdir -p /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + + # Set up Docker repository + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + + # Install latest Docker Engine + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin # Ensure Docker is running sudo systemctl start docker From 9e61fb8e5f31b0a5e5d4df213d6ed133819c4091 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 17:25:16 +0800 Subject: [PATCH 14/20] fix: fix docker --- .../build_musllinux_arm64_wheels.yml | 27 +++---------------- 1 file changed, 4 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 748cb85b0c2..eb598f39bfe 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -26,24 +26,6 @@ jobs: steps: - name: Setup Docker permissions run: | - # Remove old Docker installation - sudo apt-get remove -y docker docker-engine docker.io containerd runc || true - - # Install Docker from official repository - sudo apt-get update - sudo apt-get install -y ca-certificates curl gnupg lsb-release - - # Add Docker's official GPG key - sudo mkdir -p /etc/apt/keyrings - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - - # Set up Docker repository - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - - # Install latest Docker Engine - sudo apt-get update - sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin - # Ensure Docker is running sudo systemctl start docker sudo systemctl enable docker @@ -71,11 +53,9 @@ jobs: git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_2_aarch64 - options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} - run: | + run: | + docker run --rm -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} \ + quay.io/pypa/musllinux_1_2_aarch64 /bin/sh -c ' cd /workspace # Configure git safe directory in container @@ -354,6 +334,7 @@ jobs: # Show final results echo "=== Final wheel files ===" ls -la ./dist/ + ' continue-on-error: false # Check test success before upload - name: Verify test completion From e5aa6fd0b8819ae33aec8e7702c5ccaad8ef842c Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 18:28:07 +0800 Subject: [PATCH 15/20] fix: fix docker --- .github/workflows/build_linux_arm64_wheels-gh.yml | 2 ++ .github/workflows/build_musllinux_arm64_wheels.yml | 8 ++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 1716d6cf399..9ab2293d035 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -67,6 +67,8 @@ jobs: continue-on-error: true - name: Setup pyenv run: | + # Remove existing pyenv installation if present + rm -rf $HOME/.pyenv curl https://pyenv.run | bash export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index eb598f39bfe..7b9b41de218 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -64,11 +64,11 @@ jobs: echo "=== Configure git safe directory ===" git config --global --add safe.directory /workspace git describe --tags - python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + python3 -c 'import sys; sys.path.append("."); from setup import get_latest_git_tag; print("version:", get_latest_git_tag())' # 1. Check system info echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'\"' -f2)" if [ -f /lib/ld-musl-aarch64.so.1 ]; then echo "musl libc aarch64" elif [ -f /lib/libc.musl-aarch64.so.1 ]; then @@ -157,7 +157,7 @@ jobs: # Update version for release (if triggered by tag) if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then pyenv shell 3.9 - + # Install bump-my-version python -m pip install bump-my-version TAG_NAME=${GITHUB_REF#refs/tags/v} @@ -276,7 +276,7 @@ jobs: echo "Installation completed at: $(date)" echo "Running basic query test..." - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + python -c 'import chdb; res = chdb.query("select 1112222222,555", "CSV"); print("Python test:", res)' echo "Running full test suite..." if make test; then From f1ceb8ddf7d33abbef6e85f3d4848fdf2b232e28 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 18:34:20 +0800 Subject: [PATCH 16/20] fix: fix docker --- .github/workflows/build_musllinux_arm64_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 7b9b41de218..c6a8f833bdb 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -64,7 +64,7 @@ jobs: echo "=== Configure git safe directory ===" git config --global --add safe.directory /workspace git describe --tags - python3 -c 'import sys; sys.path.append("."); from setup import get_latest_git_tag; print("version:", get_latest_git_tag())' + python3 -c "import sys; sys.path.append(\".\"); from setup import get_latest_git_tag; print(\"version:\", get_latest_git_tag())" # 1. Check system info echo "=== Container System Info ===" From d3fbd0498a225b0d8316cae9dd1ff16bd2d69f8b Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Thu, 20 Nov 2025 18:43:26 +0800 Subject: [PATCH 17/20] fix: fix docker --- .github/workflows/build_musllinux_arm64_wheels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index c6a8f833bdb..50ba4b880e2 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -276,7 +276,7 @@ jobs: echo "Installation completed at: $(date)" echo "Running basic query test..." - python -c 'import chdb; res = chdb.query("select 1112222222,555", "CSV"); print("Python test:", res)' + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print('Python test:', res)" echo "Running full test suite..." if make test; then From 2d75be01a83d44c744f712f71ab06dc6ede47b56 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Fri, 21 Nov 2025 00:17:24 +0800 Subject: [PATCH 18/20] chore: update workflow --- .../build_musllinux_arm64_wheels.yml | 26 +------------------ 1 file changed, 1 insertion(+), 25 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 50ba4b880e2..8b0958e3722 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -276,7 +276,7 @@ jobs: echo "Installation completed at: $(date)" echo "Running basic query test..." - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print('Python test:', res)" + python -c "import chdb; res = chdb.query(\"select 1112222222,555\", \"CSV\"); print(\"Python test:\", res)" echo "Running full test suite..." if make test; then @@ -307,30 +307,6 @@ jobs: touch /workspace/.test_success_marker echo "Test success marker created at: $(date)" - # 8. Scan chdb libraries - echo "=== Scanning chdb libraries ===" - FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - # Show final results echo "=== Final wheel files ===" ls -la ./dist/ From 1636fcfa7632f4370343a461eb129f981537c48c Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Fri, 21 Nov 2025 14:04:32 +0800 Subject: [PATCH 19/20] fix: fix docker --- .github/workflows/build_musllinux_arm64_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index f7faabc1cc3..f9440cd4c8b 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -70,7 +70,7 @@ jobs: # 1. Check system info echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'\"' -f2)" + echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '\"')" if [ -f /lib/ld-musl-aarch64.so.1 ]; then echo "musl libc aarch64" elif [ -f /lib/libc.musl-aarch64.so.1 ]; then @@ -259,7 +259,7 @@ jobs: echo "Wheel file details:" file dist/*.whl - TOTAL_TESTS=5 + TOTAL_TESTS=6 CURRENT_TEST=0 TEST_FAILED=false From 3ecfa4bf93f3d7f5e5e6de0bf6481b3f70290257 Mon Sep 17 00:00:00 2001 From: wudidapaopao Date: Fri, 21 Nov 2025 14:33:13 +0800 Subject: [PATCH 20/20] fix: fix docker --- .github/scripts/build-musllinux-arm64.sh | 281 ++++++++++++++++++ .../build_musllinux_arm64_wheels.yml | 257 +--------------- 2 files changed, 282 insertions(+), 256 deletions(-) create mode 100644 .github/scripts/build-musllinux-arm64.sh diff --git a/.github/scripts/build-musllinux-arm64.sh b/.github/scripts/build-musllinux-arm64.sh new file mode 100644 index 00000000000..265ea487c1b --- /dev/null +++ b/.github/scripts/build-musllinux-arm64.sh @@ -0,0 +1,281 @@ +#!/bin/sh +set -e + +cd /workspace + +# Configure git safe directory in container +apk update +apk add --no-cache git python3 py3-pip py3-setuptools +echo "=== Configure git safe directory ===" +git config --global --add safe.directory /workspace +git describe --tags +python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + +# 1. Check system info +echo "=== Container System Info ===" +echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +else + echo "Not musl libc" +fi +echo "Workspace mounted at: /workspace" +ls -la /workspace + +# 2. Install build dependencies +echo "=== Installing build dependencies ===" +apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers +apk add --no-cache make cmake ccache ninja yasm gawk +apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + +# 3. Scan SQLite vulnerabilities +echo "=== Scanning SQLite vulnerabilities ===" +# Install grype +curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +grype db update + +# Check SQLite vulnerabilities +echo "Scanning SQLite packages for vulnerabilities..." +GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +echo "Raw grype output:" +echo "$GRYPE_RAW_OUTPUT" + +SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" +else + echo "No SQLite vulnerabilities found" +fi + +# 4. Setup Python environments +echo "=== Setting up Python environments ===" +# Setup pyenv +curl https://pyenv.run | bash +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" + +# Install Python versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing Python $version" + pyenv install $version:latest +done +pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 + +# Verify installations +echo "Installed versions:" +pyenv versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" +done +echo "All Python versions verified successfully!" + +# Install Rust +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +source $HOME/.cargo/env +rustup toolchain install nightly-2025-07-07 +rustup component add --toolchain nightly-2025-07-07 rust-src + +# Install Python dependencies +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset +done + +# Update version for release (if triggered by tag) +if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then + pyenv shell 3.9 + + # Install bump-my-version + python -m pip install bump-my-version + TAG_NAME=${GITHUB_REF#refs/tags/v} + bump-my-version replace --new-version $TAG_NAME + echo "Version files updated to $TAG_NAME" + pyenv shell --unset +fi + +# 5. Build chdb +echo "=== Building chdb ===" +echo "Timestamp: $(date)" +echo "Current directory: $(pwd)" +echo "Available disk space: $(df -h .)" + +# Setup clang +echo "Setting up clang compiler..." +ln -sf /usr/bin/clang-20 /usr/bin/clang +ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ +echo "Compiler versions:" +$CC --version +$CXX --version + +# Build +echo "Starting chdb build with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Build start time: $(date)" +bash ./chdb/build-musl.sh +echo "Build end time: $(date)" + +# Test +echo "Running smoke test with Python 3.9..." +pyenv shell 3.9 +python --version +echo "Test start time: $(date)" +bash -x ./chdb/test_smoke.sh +echo "Test end time: $(date)" + +# Check build results +echo "Build results summary:" +ccache -s +echo "chdb directory contents:" +ls -lh chdb +echo "Build artifacts size:" +du -sh chdb + +# 6. Create and audit wheels +echo "=== Creating and auditing wheels ===" +echo "Wheel creation start time: $(date)" +echo "Available disk space before wheel build: $(df -h .)" + +# Build wheels +echo "Building wheels with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Running make wheel..." +make wheel +echo "Wheel build completed at: $(date)" +echo "Initial wheel files:" +ls -lh dist/ || echo "No dist directory yet" + +# Install patchelf +echo "Installing patchelf for wheel auditing..." +wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +tar -xvf patchelf.tar.gz +cp bin/patchelf /usr/bin/ +chmod +x /usr/bin/patchelf +echo "patchelf version: $(patchelf --version)" + +# Audit wheels +echo "Auditing wheels with Python 3.13..." +pyenv shell 3.13 +python --version +python -m pip install auditwheel +echo "auditwheel version: $(auditwheel --version)" +echo "Starting wheel audit at: $(date)" +auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl +echo "Wheel audit completed at: $(date)" + +# Clean up non-musllinux wheels +echo "Cleaning up non-musllinux wheels..." +echo "Before cleanup:" +ls -lh dist/ +rm -f dist/*-linux_aarch64.whl +echo "After cleanup:" +ls -lh dist/ +echo "Final wheel sizes:" +du -sh dist/* + +# 7. Test wheels +echo "=== Testing wheels ===" +echo "Wheel testing start time: $(date)" +echo "Available wheels for testing:" +ls -lh dist/*.whl +echo "Wheel file details:" +file dist/*.whl + +TOTAL_TESTS=6 +CURRENT_TEST=0 +TEST_FAILED=false + +for version in 3.9 3.10 3.11 3.12 3.13 3.14; do + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." + pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." + python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Running basic query test..." + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi + + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" +done + +echo "All wheel tests completed at: $(date)" + +# Check if any tests failed +if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 +fi + +# Create test success marker file only if all tests passed +echo "All tests passed successfully!" +echo "Creating test success marker..." +touch /workspace/.test_success_marker +echo "Test success marker created at: $(date)" + +# 8. Scan chdb libraries +echo "=== Scanning chdb libraries ===" +FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +SQLITE_VULNERABILITIES_FOUND=false + +for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi +done + +if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" +else + echo "No SQLite vulnerabilities found in chdb libraries" +fi + +# Show final results +echo "=== Final wheel files ===" +ls -la ./dist/ \ No newline at end of file diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index f9440cd4c8b..f3e5c34939f 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -57,262 +57,7 @@ jobs: - name: Build chdb wheels in container run: | docker run --rm -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} \ - quay.io/pypa/musllinux_1_2_aarch64 /bin/sh -c ' - cd /workspace - - # Configure git safe directory in container - apk update - apk add --no-cache git python3 py3-pip py3-setuptools - echo "=== Configure git safe directory ===" - git config --global --add safe.directory /workspace - git describe --tags - python3 -c "import sys; sys.path.append(\".\"); from setup import get_latest_git_tag; print(\"version:\", get_latest_git_tag())" - - # 1. Check system info - echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d= -f2 | tr -d '\"')" - if [ -f /lib/ld-musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - else - echo "Not musl libc" - fi - echo "Workspace mounted at: /workspace" - ls -la /workspace - - # 2. Install build dependencies - echo "=== Installing build dependencies ===" - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - apk add --no-cache make cmake ccache ninja yasm gawk - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - - # 3. Scan SQLite vulnerabilities - echo "=== Scanning SQLite vulnerabilities ===" - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - grype db update - - # Check SQLite vulnerabilities - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - - # 4. Setup Python environments - echo "=== Setting up Python environments ===" - # Setup pyenv - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - - # Install Python versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing Python $version" - pyenv install $version:latest - done - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - - # Verify installations - echo "Installed versions:" - pyenv versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - - # Install Python dependencies - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - # Update version for release (if triggered by tag) - if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then - pyenv shell 3.9 - - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - fi - - # 5. Build chdb - echo "=== Building chdb ===" - echo "Timestamp: $(date)" - echo "Current directory: $(pwd)" - echo "Available disk space: $(df -h .)" - - # Setup clang - echo "Setting up clang compiler..." - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - echo "Compiler versions:" - $CC --version - $CXX --version - - # Build - echo "Starting chdb build with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Build start time: $(date)" - bash ./chdb/build-musl.sh - echo "Build end time: $(date)" - - # Test - echo "Running smoke test with Python 3.9..." - pyenv shell 3.9 - python --version - echo "Test start time: $(date)" - bash -x ./chdb/test_smoke.sh - echo "Test end time: $(date)" - - # Check build results - echo "Build results summary:" - ccache -s - echo "chdb directory contents:" - ls -lh chdb - echo "Build artifacts size:" - du -sh chdb - - # 6. Create and audit wheels - echo "=== Creating and auditing wheels ===" - echo "Wheel creation start time: $(date)" - echo "Available disk space before wheel build: $(df -h .)" - - # Build wheels - echo "Building wheels with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Running make wheel..." - make wheel - echo "Wheel build completed at: $(date)" - echo "Initial wheel files:" - ls -lh dist/ || echo "No dist directory yet" - - # Install patchelf - echo "Installing patchelf for wheel auditing..." - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - echo "patchelf version: $(patchelf --version)" - - # Audit wheels - echo "Auditing wheels with Python 3.13..." - pyenv shell 3.13 - python --version - python -m pip install auditwheel - echo "auditwheel version: $(auditwheel --version)" - echo "Starting wheel audit at: $(date)" - auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - echo "Wheel audit completed at: $(date)" - - # Clean up non-musllinux wheels - echo "Cleaning up non-musllinux wheels..." - echo "Before cleanup:" - ls -lh dist/ - rm -f dist/*-linux_aarch64.whl - echo "After cleanup:" - ls -lh dist/ - echo "Final wheel sizes:" - du -sh dist/* - - # 7. Test wheels - echo "=== Testing wheels ===" - echo "Wheel testing start time: $(date)" - echo "Available wheels for testing:" - ls -lh dist/*.whl - echo "Wheel file details:" - file dist/*.whl - - TOTAL_TESTS=6 - CURRENT_TEST=0 - TEST_FAILED=false - - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - CURRENT_TEST=$((CURRENT_TEST + 1)) - echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" - echo "Test start time: $(date)" - - echo "Switching to Python $version..." - pyenv shell $version - python --version - echo "pip version: $(python -m pip --version)" - - echo "Installing chdb wheel..." - python -m pip install dist/*.whl --force-reinstall - echo "Installation completed at: $(date)" - - echo "Running basic query test..." - python -c "import chdb; res = chdb.query(\"select 1112222222,555\", \"CSV\"); print(\"Python test:\", res)" - - echo "Running full test suite..." - if make test; then - echo "Test suite PASSED for Python $version at: $(date)" - else - echo "Test suite FAILED for Python $version at: $(date)" - TEST_FAILED=true - break - fi - - pyenv shell --unset - echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" - echo "" - done - - echo "All wheel tests completed at: $(date)" - - # Check if any tests failed - if [ "$TEST_FAILED" = true ]; then - echo "ERROR: One or more test suites failed!" - echo "Test failure detected - aborting build process" - exit 1 - fi - - # Create test success marker file only if all tests passed - echo "All tests passed successfully!" - echo "Creating test success marker..." - touch /workspace/.test_success_marker - echo "Test success marker created at: $(date)" - - # Show final results - echo "=== Final wheel files ===" - ls -la ./dist/ - ' + quay.io/pypa/musllinux_1_2_aarch64 /bin/sh /workspace/.github/scripts/build-musllinux-arm64.sh continue-on-error: false # Check test success before upload - name: Verify test completion