diff --git a/.github/scripts/build-musllinux-arm64.sh b/.github/scripts/build-musllinux-arm64.sh new file mode 100644 index 00000000000..265ea487c1b --- /dev/null +++ b/.github/scripts/build-musllinux-arm64.sh @@ -0,0 +1,281 @@ +#!/bin/sh +set -e + +cd /workspace + +# Configure git safe directory in container +apk update +apk add --no-cache git python3 py3-pip py3-setuptools +echo "=== Configure git safe directory ===" +git config --global --add safe.directory /workspace +git describe --tags +python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" + +# 1. Check system info +echo "=== Container System Info ===" +echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +if [ -f /lib/ld-musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +elif [ -f /lib/libc.musl-aarch64.so.1 ]; then + echo "musl libc aarch64" +else + echo "Not musl libc" +fi +echo "Workspace mounted at: /workspace" +ls -la /workspace + +# 2. Install build dependencies +echo "=== Installing build dependencies ===" +apk add --no-cache make build-base openssl-dev zlib-dev \ + bzip2-dev readline-dev sqlite-dev wget curl llvm \ + ncurses-dev xz-dev tk-dev libxml2-dev \ + libffi-dev linux-headers +apk add --no-cache make cmake ccache ninja yasm gawk +apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 + +# 3. Scan SQLite vulnerabilities +echo "=== Scanning SQLite vulnerabilities ===" +# Install grype +curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +grype db update + +# Check SQLite vulnerabilities +echo "Scanning SQLite packages for vulnerabilities..." +GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +echo "Raw grype output:" +echo "$GRYPE_RAW_OUTPUT" + +SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +if [ -n "$SQLITE_SCAN_OUTPUT" ]; then + echo "SQLite vulnerabilities found in packages! Build should be reviewed." + echo "SQLite vulnerability details:" + echo "$SQLITE_SCAN_OUTPUT" +else + echo "No SQLite vulnerabilities found" +fi + +# 4. Setup Python environments +echo "=== Setting up Python environments ===" +# Setup pyenv +curl https://pyenv.run | bash +export PATH="$HOME/.pyenv/bin:$PATH" +eval "$(pyenv init -)" + +# Install Python versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing Python $version" + pyenv install $version:latest +done +pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 + +# Verify installations +echo "Installed versions:" +pyenv versions +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + if ! pyenv versions --bare | grep -q "^$version"; then + echo "ERROR: Python $version is not installed!" + exit 1 + fi + echo "Python $version is installed" +done +echo "All Python versions verified successfully!" + +# Install Rust +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +source $HOME/.cargo/env +rustup toolchain install nightly-2025-07-07 +rustup component add --toolchain nightly-2025-07-07 rust-src + +# Install Python dependencies +for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do + echo "Installing dependencies for Python $version" + pyenv shell $version + python -m pip install --upgrade pip + if [ "$version" = "3.8" ]; then + python -m pip install setuptools tox twine psutil wheel + else + python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel + fi + pyenv shell --unset +done + +# Update version for release (if triggered by tag) +if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then + pyenv shell 3.9 + + # Install bump-my-version + python -m pip install bump-my-version + TAG_NAME=${GITHUB_REF#refs/tags/v} + bump-my-version replace --new-version $TAG_NAME + echo "Version files updated to $TAG_NAME" + pyenv shell --unset +fi + +# 5. Build chdb +echo "=== Building chdb ===" +echo "Timestamp: $(date)" +echo "Current directory: $(pwd)" +echo "Available disk space: $(df -h .)" + +# Setup clang +echo "Setting up clang compiler..." +ln -sf /usr/bin/clang-20 /usr/bin/clang +ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +export CC=/usr/bin/clang +export CXX=/usr/bin/clang++ +echo "Compiler versions:" +$CC --version +$CXX --version + +# Build +echo "Starting chdb build with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Build start time: $(date)" +bash ./chdb/build-musl.sh +echo "Build end time: $(date)" + +# Test +echo "Running smoke test with Python 3.9..." +pyenv shell 3.9 +python --version +echo "Test start time: $(date)" +bash -x ./chdb/test_smoke.sh +echo "Test end time: $(date)" + +# Check build results +echo "Build results summary:" +ccache -s +echo "chdb directory contents:" +ls -lh chdb +echo "Build artifacts size:" +du -sh chdb + +# 6. Create and audit wheels +echo "=== Creating and auditing wheels ===" +echo "Wheel creation start time: $(date)" +echo "Available disk space before wheel build: $(df -h .)" + +# Build wheels +echo "Building wheels with Python 3.8..." +pyenv shell 3.8 +python --version +echo "Running make wheel..." +make wheel +echo "Wheel build completed at: $(date)" +echo "Initial wheel files:" +ls -lh dist/ || echo "No dist directory yet" + +# Install patchelf +echo "Installing patchelf for wheel auditing..." +wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz +tar -xvf patchelf.tar.gz +cp bin/patchelf /usr/bin/ +chmod +x /usr/bin/patchelf +echo "patchelf version: $(patchelf --version)" + +# Audit wheels +echo "Auditing wheels with Python 3.13..." +pyenv shell 3.13 +python --version +python -m pip install auditwheel +echo "auditwheel version: $(auditwheel --version)" +echo "Starting wheel audit at: $(date)" +auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl +echo "Wheel audit completed at: $(date)" + +# Clean up non-musllinux wheels +echo "Cleaning up non-musllinux wheels..." +echo "Before cleanup:" +ls -lh dist/ +rm -f dist/*-linux_aarch64.whl +echo "After cleanup:" +ls -lh dist/ +echo "Final wheel sizes:" +du -sh dist/* + +# 7. Test wheels +echo "=== Testing wheels ===" +echo "Wheel testing start time: $(date)" +echo "Available wheels for testing:" +ls -lh dist/*.whl +echo "Wheel file details:" +file dist/*.whl + +TOTAL_TESTS=6 +CURRENT_TEST=0 +TEST_FAILED=false + +for version in 3.9 3.10 3.11 3.12 3.13 3.14; do + CURRENT_TEST=$((CURRENT_TEST + 1)) + echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" + echo "Test start time: $(date)" + + echo "Switching to Python $version..." + pyenv shell $version + python --version + echo "pip version: $(python -m pip --version)" + + echo "Installing chdb wheel..." + python -m pip install dist/*.whl --force-reinstall + echo "Installation completed at: $(date)" + + echo "Running basic query test..." + python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" + + echo "Running full test suite..." + if make test; then + echo "Test suite PASSED for Python $version at: $(date)" + else + echo "Test suite FAILED for Python $version at: $(date)" + TEST_FAILED=true + break + fi + + pyenv shell --unset + echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" + echo "" +done + +echo "All wheel tests completed at: $(date)" + +# Check if any tests failed +if [ "$TEST_FAILED" = true ]; then + echo "ERROR: One or more test suites failed!" + echo "Test failure detected - aborting build process" + exit 1 +fi + +# Create test success marker file only if all tests passed +echo "All tests passed successfully!" +echo "Creating test success marker..." +touch /workspace/.test_success_marker +echo "Test success marker created at: $(date)" + +# 8. Scan chdb libraries +echo "=== Scanning chdb libraries ===" +FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +SQLITE_VULNERABILITIES_FOUND=false + +for file in $FILES_TO_SCAN; do + if [ -f "$file" ]; then + echo "=== Scanning $file ===" + SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + echo "$SCAN_OUTPUT" + + if echo "$SCAN_OUTPUT" | grep -qi sqlite; then + echo "SQLite vulnerability found in $file" + SQLITE_VULNERABILITIES_FOUND=true + fi + fi +done + +if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then + echo "SQLite vulnerabilities detected in chdb libraries!" +else + echo "No SQLite vulnerabilities found in chdb libraries" +fi + +# Show final results +echo "=== Final wheel files ===" +ls -la ./dist/ \ No newline at end of file diff --git a/.github/workflows/build_linux_arm64_wheels-gh.yml b/.github/workflows/build_linux_arm64_wheels-gh.yml index 4c9c77bb763..af1b6295bbe 100644 --- a/.github/workflows/build_linux_arm64_wheels-gh.yml +++ b/.github/workflows/build_linux_arm64_wheels-gh.yml @@ -24,7 +24,7 @@ on: jobs: build_universal_wheel: name: Build Universal Wheel (Linux ARM64) - runs-on: GH-Linux-ARM64 + runs-on: [self-hosted, linux, arm64, ubuntu-latest] if: ${{ !github.event.pull_request.draft }} steps: - name: Install Python build dependencies @@ -33,11 +33,13 @@ jobs: sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev + libffi-dev liblzma-dev golang-go - name: Scan SQLite vulnerabilities with grype run: | # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin + mkdir -p $HOME/.local/bin + curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b $HOME/.local/bin + echo "$HOME/.local/bin" >> $GITHUB_PATH sudo apt-get update && sudo apt-get install -y jq lsb-release # Detect OS distribution info @@ -46,11 +48,11 @@ jobs: echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" # Update grype vulnerability database - grype db update + $HOME/.local/bin/grype db update # Check SQLite vulnerabilities in installed packages echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) + GRYPE_RAW_OUTPUT=$($HOME/.local/bin/grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) echo "Raw grype output:" echo "$GRYPE_RAW_OUTPUT" @@ -67,6 +69,8 @@ jobs: continue-on-error: true - name: Setup pyenv run: | + # Remove existing pyenv installation if present + rm -rf $HOME/.pyenv curl https://pyenv.run | bash export PATH="$HOME/.pyenv/bin:$PATH" eval "$(pyenv init -)" @@ -213,7 +217,7 @@ jobs: for file in $FILES_TO_SCAN; do if [ -f "$file" ]; then echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) + SCAN_OUTPUT=$($HOME/.local/bin/grype "$file" 2>/dev/null || true) echo "$SCAN_OUTPUT" if echo "$SCAN_OUTPUT" | grep -qi sqlite; then @@ -256,7 +260,10 @@ jobs: patchelf --version - name: Audit wheels run: | - python3 -m pip install auditwheel + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + pyenv shell 3.9 + python -m pip install auditwheel auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl continue-on-error: false - name: Show files diff --git a/.github/workflows/build_linux_x86_wheels.yml b/.github/workflows/build_linux_x86_wheels.yml index 744ef0e0795..d808cc68be6 100644 --- a/.github/workflows/build_linux_x86_wheels.yml +++ b/.github/workflows/build_linux_x86_wheels.yml @@ -1,366 +1,366 @@ -name: Build Linux X86 +# name: Build Linux X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (Linux x86_64) - runs-on: gh-64c - if: ${{ !github.event.pull_request.draft }} - steps: - - name: Install Python build dependencies - run: | - sudo apt-get update - sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ - libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ - libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ - libffi-dev liblzma-dev - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - sudo apt-get update && sudo apt-get install -y jq lsb-release +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (Linux x86_64) +# runs-on: gh-64c +# if: ${{ !github.event.pull_request.draft }} +# steps: +# - name: Install Python build dependencies +# run: | +# sudo apt-get update +# sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ +# libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ +# libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ +# libffi-dev liblzma-dev +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# sudo apt-get update && sudo apt-get install -y jq lsb-release - # Detect OS distribution info - DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') - DISTRO_VERSION=$(lsb_release -sr) - echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" +# # Detect OS distribution info +# DISTRO_ID=$(lsb_release -si | tr '[:upper:]' '[:lower:]') +# DISTRO_VERSION=$(lsb_release -sr) +# echo "Detected OS: $DISTRO_ID:$DISTRO_VERSION" - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/var/lib/dpkg --distro "$DISTRO_ID:$DISTRO_VERSION" --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: true - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: true +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel jupyter nbconvert - pyenv shell --unset - done - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - name: Install clang++ for Ubuntu - run: | - pwd - uname -a - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 19 - which clang++-19 - clang++-19 --version - sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget - # Install WebAssembly linker (wasm-ld) - sudo apt-get install -y lld-19 - # Create symlink for wasm-ld - if ! command -v wasm-ld &> /dev/null; then - sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true - fi - which wasm-ld || echo "wasm-ld not found in PATH" - ccache -s - - name: Update git - run: | - sudo add-apt-repository ppa:git-core/ppa -y - sudo apt-get update - sudo apt-get install -y git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel jupyter nbconvert +# pyenv shell --unset +# done +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - name: Install clang++ for Ubuntu +# run: | +# pwd +# uname -a +# wget https://apt.llvm.org/llvm.sh +# chmod +x llvm.sh +# sudo ./llvm.sh 19 +# which clang++-19 +# clang++-19 --version +# sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget +# # Install WebAssembly linker (wasm-ld) +# sudo apt-get install -y lld-19 +# # Create symlink for wasm-ld +# if ! command -v wasm-ld &> /dev/null; then +# sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true +# fi +# which wasm-ld || echo "wasm-ld not found in PATH" +# ccache -s +# - name: Update git +# run: | +# sudo add-apt-repository ppa:git-core/ppa -y +# sudo apt-get update +# sudo apt-get install -y git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ubuntu-22.04-x86_64 - max-size: 5G - append-timestamp: true - - name: remove old clang and link clang-19 to clang - run: | - sudo rm -f /usr/bin/clang || true - sudo ln -s /usr/bin/clang-19 /usr/bin/clang - sudo rm -f /usr/bin/clang++ || true - sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - export PATH="$HOME/.pyenv/bin:$PATH" - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ubuntu-22.04-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: remove old clang and link clang-19 to clang +# run: | +# sudo rm -f /usr/bin/clang || true +# sudo ln -s /usr/bin/clang-19 /usr/bin/clang +# sudo rm -f /usr/bin/clang++ || true +# sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# export PATH="$HOME/.pyenv/bin:$PATH" +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - sudo cp bin/patchelf /usr/bin/ - sudo chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - python3 -m pip install auditwheel - auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - sudo rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Setup core dump collection - run: | - mkdir -p tmp/core - echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern - ulimit -c unlimited - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-linux-x86_64.tar.gz" - ls -lh core-files-linux-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-linux-x86_64 - path: core-files-linux-x86_64.tar.gz - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-linux-x86_64 - path: | - ./dist/*.whl - ./linux-x86_64-libchdb.tar.gz - ./linux-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# sudo cp bin/patchelf /usr/bin/ +# sudo chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# python3 -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat manylinux2014_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# sudo rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# echo "tmp/core/core.%p" | sudo tee /proc/sys/kernel/core_pattern +# ulimit -c unlimited +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-linux-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-linux-x86_64.tar.gz" +# ls -lh core-files-linux-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-linux-x86_64 +# path: core-files-linux-x86_64.tar.gz +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf linux-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} linux-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-linux-x86_64 +# path: | +# ./dist/*.whl +# ./linux-x86_64-libchdb.tar.gz +# ./linux-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml index 04cf1b2e4b6..dd738317120 100644 --- a/.github/workflows/build_macos_arm64_wheels.yml +++ b/.github/workflows/build_macos_arm64_wheels.yml @@ -1,365 +1,365 @@ -name: Build macOS arm64 +# name: Build macOS arm64 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS ARM64) - runs-on: macos-14-xlarge - if: ${{ !github.event.pull_request.draft }} - steps: - - name: Check machine architecture - run: | - echo "=== Machine Architecture Information ===" - echo "Machine type: $(uname -m)" - echo "Architecture: $(arch)" - echo "System info: $(uname -a)" - echo "Hardware info:" - system_profiler SPHardwareDataType | grep "Chip\|Processor" - if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then - echo "This is an ARM64 (Apple Silicon) machine" - else - echo "This is an x86_64 (Intel) machine" - fi - - name: Free up disk space (Initial) - run: | - # Clean Homebrew cache - brew cleanup -s 2>/dev/null || true - rm -rf "$(brew --cache)" 2>/dev/null || true - sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true - sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true - sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true - sudo rm -rf /tmp/* 2>/dev/null || true - echo "=== Disk usage after cleanup ===" - df -h - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS ARM64) +# runs-on: macos-14-xlarge +# if: ${{ !github.event.pull_request.draft }} +# steps: +# - name: Check machine architecture +# run: | +# echo "=== Machine Architecture Information ===" +# echo "Machine type: $(uname -m)" +# echo "Architecture: $(arch)" +# echo "System info: $(uname -a)" +# echo "Hardware info:" +# system_profiler SPHardwareDataType | grep "Chip\|Processor" +# if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then +# echo "This is an ARM64 (Apple Silicon) machine" +# else +# echo "This is an x86_64 (Intel) machine" +# fi +# - name: Free up disk space (Initial) +# run: | +# # Clean Homebrew cache +# brew cleanup -s 2>/dev/null || true +# rm -rf "$(brew --cache)" 2>/dev/null || true +# sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true +# sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true +# sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true +# sudo rm -rf /tmp/* 2>/dev/null || true +# echo "=== Disk usage after cleanup ===" +# df -h +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd - brew install openssl@3 || echo "OpenSSL install failed, continuing..." - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which wasm-ld || echo "wasm-ld not found in PATH" - which go - go version - ccache -s | echo "ccache not available yet" - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal readline sqlite xz z3 zstd +# brew install openssl@3 || echo "OpenSSL install failed, continuing..." +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which wasm-ld || echo "wasm-ld not found in PATH" +# which go +# go version +# ccache -s | echo "ccache not available yet" +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: Free up disk space (Before compilation) - run: | - echo "=== Disk usage before compilation cleanup ===" - df -h - brew cleanup -s 2>/dev/null || true - rm -rf "$(brew --cache)" 2>/dev/null || true - rm -rf ~/.cache/pip 2>/dev/null || true - rm -rf ~/.pyenv/.cache 2>/dev/null || true - rm -rf ~/.cargo/registry/cache 2>/dev/null || true - echo "=== Disk usage after cleanup ===" - df -h - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ runner.os }}-arm64-ccache - max-size: 5G - append-timestamp: true - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.8 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.8 - bash -x ./chdb/test_smoke.sh - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - source ~/.cargo/env - eval "$(pyenv init -)" - pyenv shell 3.8 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Build wheels - run: | - rm -rf chdb/build/ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl - - name: Verify wheel sizes - run: | - echo "=== Wheel sizes ===" - du -sh dist/* - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Free up disk space - run: | - # Clean more build artifacts - rm -rf buildlib/contrib 2>/dev/null || true - rm -rf buildlib/base 2>/dev/null || true - rm -rf buildlib/src 2>/dev/null || true - - name: Test wheel on all Python versions - run: | - ulimit -c unlimited - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall --no-cache-dir - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - python -m pip uninstall -y chdb - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-arm64.tar.gz" - ls -lh core-files-macos-arm64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-arm64 - path: core-files-macos-arm64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-arm64 - path: | - ./dist/*.whl - ./macos-arm64-libchdb.tar.gz - ./macos-arm64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.8 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: Free up disk space (Before compilation) +# run: | +# echo "=== Disk usage before compilation cleanup ===" +# df -h +# brew cleanup -s 2>/dev/null || true +# rm -rf "$(brew --cache)" 2>/dev/null || true +# rm -rf ~/.cache/pip 2>/dev/null || true +# rm -rf ~/.pyenv/.cache 2>/dev/null || true +# rm -rf ~/.cargo/registry/cache 2>/dev/null || true +# echo "=== Disk usage after cleanup ===" +# df -h +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ${{ runner.os }}-arm64-ccache +# max-size: 5G +# append-timestamp: true +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.8 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.8 +# bash -x ./chdb/test_smoke.sh +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# source ~/.cargo/env +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Build wheels +# run: | +# rm -rf chdb/build/ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl +# - name: Verify wheel sizes +# run: | +# echo "=== Wheel sizes ===" +# du -sh dist/* +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Free up disk space +# run: | +# # Clean more build artifacts +# rm -rf buildlib/contrib 2>/dev/null || true +# rm -rf buildlib/base 2>/dev/null || true +# rm -rf buildlib/src 2>/dev/null || true +# - name: Test wheel on all Python versions +# run: | +# ulimit -c unlimited +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall --no-cache-dir +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# python -m pip uninstall -y chdb +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-arm64.tar.gz" +# ls -lh core-files-macos-arm64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-arm64 +# path: core-files-macos-arm64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-arm64 +# path: | +# ./dist/*.whl +# ./macos-arm64-libchdb.tar.gz +# ./macos-arm64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.8 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml index 1791a9fefa5..d84a64076ab 100644 --- a/.github/workflows/build_macos_x86_wheels.yml +++ b/.github/workflows/build_macos_x86_wheels.yml @@ -1,365 +1,365 @@ -name: Build macOS X86 +# name: Build macOS X86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_universal_wheel: - name: Build Universal Wheel (macOS x86_64) - runs-on: macos-15-intel - if: ${{ !github.event.pull_request.draft }} - timeout-minutes: 600 - steps: - - name: Check machine architecture - run: | - echo "=== Machine Architecture Information ===" - echo "Machine type: $(uname -m)" - echo "Architecture: $(arch)" - echo "System info: $(uname -a)" - echo "Hardware info:" - system_profiler SPHardwareDataType | grep "Chip\|Processor" - if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then - echo "This is an ARM64 (Apple Silicon) machine" - else - echo "This is an x86_64 (Intel) machine" - fi - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" +# jobs: +# build_universal_wheel: +# name: Build Universal Wheel (macOS x86_64) +# runs-on: macos-15-intel +# if: ${{ !github.event.pull_request.draft }} +# timeout-minutes: 600 +# steps: +# - name: Check machine architecture +# run: | +# echo "=== Machine Architecture Information ===" +# echo "Machine type: $(uname -m)" +# echo "Architecture: $(arch)" +# echo "System info: $(uname -a)" +# echo "Hardware info:" +# system_profiler SPHardwareDataType | grep "Chip\|Processor" +# if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then +# echo "This is an ARM64 (Apple Silicon) machine" +# else +# echo "This is an x86_64 (Intel) machine" +# fi +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.9 3.10 3.11 3.12 3.13 3.14 +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.9 3.10 3.11 3.12 3.13 3.14 - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Installed Python versions:" - pyenv versions - echo "" - echo "Verifying all required Python versions are available:" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "✓ Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert - pyenv shell --unset - done - - name: Remove /usr/local/bin/python3 - run: | - sudo rm -f /usr/local/bin/python3 - - name: Install clang++ for macOS - run: | - pwd - uname -a - export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 - brew update - brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd - brew install --ignore-dependencies llvm@19 - brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv - brew install ccache || echo "ccache installation failed, continuing without it" - brew install go - cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH - which clang++ - clang++ --version - which go - go version - ccache -s || echo "ccache not available yet" - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update - # Check SQLite vulnerabilities in Homebrew packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - exit 1 - else - echo "✅ No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Upgrade Rust toolchain - run: | - rustup toolchain install nightly-2025-07-07 - rustup default nightly-2025-07-07 - rustup component add rust-src - rustc --version - cargo --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Installed Python versions:" +# pyenv versions +# echo "" +# echo "Verifying all required Python versions are available:" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "✓ Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert +# pyenv shell --unset +# done +# - name: Remove /usr/local/bin/python3 +# run: | +# sudo rm -f /usr/local/bin/python3 +# - name: Install clang++ for macOS +# run: | +# pwd +# uname -a +# export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 +# brew update +# brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd +# brew install --ignore-dependencies llvm@19 +# brew install git ninja libtool gettext gcc binutils grep findutils nasm lld@19 libiconv +# brew install ccache || echo "ccache installation failed, continuing without it" +# brew install go +# cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH +# which clang++ +# clang++ --version +# which go +# go version +# ccache -s || echo "ccache not available yet" +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# # Update grype vulnerability database +# grype db update +# # Check SQLite vulnerabilities in Homebrew packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/opt/homebrew --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "❌ SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Upgrade Rust toolchain +# run: | +# rustup toolchain install nightly-2025-07-07 +# rustup default nightly-2025-07-07 +# rustup component add rust-src +# rustc --version +# cargo --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: ${{ runner.os }}-x86_64-ccache - max-size: 10G - append-timestamp: true - env: - CCACHE_NOHASHDIR: "true" - - name: Run chdb/build.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - pyenv shell 3.9 - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - bash gen_manifest.sh - bash ./chdb/build.sh - pyenv shell 3.9 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Run chdb/build/build_static_lib.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.9 - bash ./chdb/build/build_static_lib.sh - pyenv shell --unset - continue-on-error: false - - name: Debug libchdb - run: | - ls -lh - llvm-nm libchdb.so | grep query_stable || true - echo "Global Symbol in libchdb.so:" - llvm-nm -g libchdb.so || true - echo "Global Symbol in libclickhouse-local-chdb.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true - echo "Global Symbol in libclickhouse-local-lib.a:" - llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true - echo "pychdb_cmd.sh:" - cat buildlib/pychdb_cmd.sh - echo "libchdb_cmd.sh:" - cat buildlib/libchdb_cmd.sh - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." - # Files to scan - FILES_TO_SCAN="" - [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" - [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "❌ SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "❌ SQLite vulnerabilities detected in chdb libraries!" - exit 1 - else - echo "✅ No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Run libchdb stub in examples dir - run: | - bash -x ./examples/runStub.sh - bash -x ./examples/runArrowTest.sh - - name: Build wheels - run: | - rm -rf chdb/build/ - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source ~/.cargo/env - export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin - export CC=$(brew --prefix llvm@19)/bin/clang - export CXX=$(brew --prefix llvm@19)/bin/clang++ - pyenv shell 3.9 - make wheel - - name: Fix wheel platform tag - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl - - name: Verify wheel sizes - run: | - echo "=== Wheel sizes ===" - du -sh dist/* - - name: Setup core dump collection - run: | - mkdir -p tmp/core - sudo sysctl kern.corefile=$PWD/tmp/core/core.%P - sudo sysctl kern.coredump=1 - ulimit -c unlimited - - name: Free up disk space - run: | - # Clean more build artifacts - rm -rf buildlib/contrib 2>/dev/null || true - rm -rf buildlib/base 2>/dev/null || true - rm -rf buildlib/src 2>/dev/null || true - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Run notebook tests - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m pip install dist/*.whl --force-reinstall - jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb - pyenv shell --unset - continue-on-error: false - - name: Check and upload core files if present - if: always() - run: | - if ls tmp/core/core.* >/dev/null 2>&1; then - echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV - tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* - echo "Core files tar created: core-files-macos-x86_64.tar.gz" - ls -lh core-files-macos-x86_64.tar.gz - else - echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV - echo "No core files found in tmp/core" - fi - continue-on-error: true - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done - - name: Upload core files artifact - if: always() && env.CORE_FILES_FOUND == 'true' - uses: actions/upload-artifact@v4 - with: - name: core-files-macos-x86_64 - path: core-files-macos-x86_64.tar.gz - - name: Show files - run: ls -lh dist - shell: bash - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Packege libchdb.so - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp - - name: Package libchdb.a - run: | - cp programs/local/chdb.h chdb.h - cp programs/local/chdb.hpp chdb.hpp - tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp - - name: Upload libchdb.so to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Upload libchdb.a to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-macos-x86_64 - path: | - ./dist/*.whl - ./macos-x86_64-libchdb.tar.gz - ./macos-x86_64-libchdb-static.tar.gz - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: ${{ runner.os }}-x86_64-ccache +# max-size: 10G +# append-timestamp: true +# env: +# CCACHE_NOHASHDIR: "true" +# - name: Run chdb/build.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# pyenv shell 3.9 +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# bash gen_manifest.sh +# bash ./chdb/build.sh +# pyenv shell 3.9 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Run chdb/build/build_static_lib.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.9 +# bash ./chdb/build/build_static_lib.sh +# pyenv shell --unset +# continue-on-error: false +# - name: Debug libchdb +# run: | +# ls -lh +# llvm-nm libchdb.so | grep query_stable || true +# echo "Global Symbol in libchdb.so:" +# llvm-nm -g libchdb.so || true +# echo "Global Symbol in libclickhouse-local-chdb.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-chdb.a || true +# echo "Global Symbol in libclickhouse-local-lib.a:" +# llvm-nm -g buildlib/programs/local/libclickhouse-local-lib.a || true +# echo "pychdb_cmd.sh:" +# cat buildlib/pychdb_cmd.sh +# echo "libchdb_cmd.sh:" +# cat buildlib/libchdb_cmd.sh +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." +# # Files to scan +# FILES_TO_SCAN="" +# [ -f libchdb.so ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.so" +# [ -f libchdb.a ] && FILES_TO_SCAN="$FILES_TO_SCAN libchdb.a" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.dylib" -o -name "*.so" \) 2>/dev/null || true)" +# SQLITE_VULNERABILITIES_FOUND=false +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "❌ SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "❌ SQLite vulnerabilities detected in chdb libraries!" +# exit 1 +# else +# echo "✅ No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Run libchdb stub in examples dir +# run: | +# bash -x ./examples/runStub.sh +# bash -x ./examples/runArrowTest.sh +# - name: Build wheels +# run: | +# rm -rf chdb/build/ +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source ~/.cargo/env +# export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin +# export CC=$(brew --prefix llvm@19)/bin/clang +# export CXX=$(brew --prefix llvm@19)/bin/clang++ +# pyenv shell 3.9 +# make wheel +# - name: Fix wheel platform tag +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl +# - name: Verify wheel sizes +# run: | +# echo "=== Wheel sizes ===" +# du -sh dist/* +# - name: Setup core dump collection +# run: | +# mkdir -p tmp/core +# sudo sysctl kern.corefile=$PWD/tmp/core/core.%P +# sudo sysctl kern.coredump=1 +# ulimit -c unlimited +# - name: Free up disk space +# run: | +# # Clean more build artifacts +# rm -rf buildlib/contrib 2>/dev/null || true +# rm -rf buildlib/base 2>/dev/null || true +# rm -rf buildlib/src 2>/dev/null || true +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Run notebook tests +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m pip install dist/*.whl --force-reinstall +# jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb +# pyenv shell --unset +# continue-on-error: false +# - name: Check and upload core files if present +# if: always() +# run: | +# if ls tmp/core/core.* >/dev/null 2>&1; then +# echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV +# tar -czvf core-files-macos-x86_64.tar.gz tmp/core/core.* +# echo "Core files tar created: core-files-macos-x86_64.tar.gz" +# ls -lh core-files-macos-x86_64.tar.gz +# else +# echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV +# echo "No core files found in tmp/core" +# fi +# continue-on-error: true +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done +# - name: Upload core files artifact +# if: always() && env.CORE_FILES_FOUND == 'true' +# uses: actions/upload-artifact@v4 +# with: +# name: core-files-macos-x86_64 +# path: core-files-macos-x86_64.tar.gz +# - name: Show files +# run: ls -lh dist +# shell: bash +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Packege libchdb.so +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp +# - name: Package libchdb.a +# run: | +# cp programs/local/chdb.h chdb.h +# cp programs/local/chdb.hpp chdb.hpp +# tar -czvf macos-x86_64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp +# - name: Upload libchdb.so to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - name: Upload libchdb.a to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} macos-x86_64-libchdb-static.tar.gz --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-macos-x86_64 +# path: | +# ./dist/*.whl +# ./macos-x86_64-libchdb.tar.gz +# ./macos-x86_64-libchdb-static.tar.gz +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} \ No newline at end of file diff --git a/.github/workflows/build_musllinux_arm64_wheels.yml b/.github/workflows/build_musllinux_arm64_wheels.yml index 591eeca9e9f..f3e5c34939f 100644 --- a/.github/workflows/build_musllinux_arm64_wheels.yml +++ b/.github/workflows/build_musllinux_arm64_wheels.yml @@ -23,9 +23,25 @@ on: jobs: build_musllinux_wheels: name: Build musllinux wheels (Alpine Linux aarch64) - runs-on: GH-Linux-ARM64 + runs-on: [self-hosted, linux, arm64, ubuntu-latest] if: ${{ !github.event.pull_request.draft }} steps: + - name: Setup Docker permissions + run: | + # Ensure Docker is running + sudo systemctl start docker + sudo systemctl enable docker + + # Add current user to docker group + sudo usermod -aG docker $USER + + # Set proper permissions on docker socket + sudo chmod 666 /var/run/docker.sock + + # Verify Docker is working + docker --version + docker info + - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -39,289 +55,9 @@ jobs: git submodule update --init --recursive --jobs 4 - name: Build chdb wheels in container - uses: addnab/docker-run-action@v3 - with: - image: quay.io/pypa/musllinux_1_2_aarch64 - options: -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} - run: | - cd /workspace - - # Configure git safe directory in container - apk update - apk add --no-cache git python3 py3-pip py3-setuptools - echo "=== Configure git safe directory ===" - git config --global --add safe.directory /workspace - git describe --tags - python3 -c "import sys; sys.path.append('.'); from setup import get_latest_git_tag; print('version:', get_latest_git_tag())" - - # 1. Check system info - echo "=== Container System Info ===" - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - elif [ -f /lib/libc.musl-aarch64.so.1 ]; then - echo "musl libc aarch64" - else - echo "Not musl libc" - fi - echo "Workspace mounted at: /workspace" - ls -la /workspace - - # 2. Install build dependencies - echo "=== Installing build dependencies ===" - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - apk add --no-cache make cmake ccache ninja yasm gawk - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - - # 3. Scan SQLite vulnerabilities - echo "=== Scanning SQLite vulnerabilities ===" - # Install grype - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - grype db update - - # Check SQLite vulnerabilities - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" - - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - - # 4. Setup Python environments - echo "=== Setting up Python environments ===" - # Setup pyenv - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - - # Install Python versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing Python $version" - pyenv install $version:latest - done - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - - # Verify installations - echo "Installed versions:" - pyenv versions - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - # Install Rust - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - - # Install Python dependencies - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - # Update version for release (if triggered by tag) - if [ "${GITHUB_REF#refs/tags/v}" != "$GITHUB_REF" ]; then - pyenv shell 3.9 - - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - fi - - # 5. Build chdb - echo "=== Building chdb ===" - echo "Timestamp: $(date)" - echo "Current directory: $(pwd)" - echo "Available disk space: $(df -h .)" - - # Setup clang - echo "Setting up clang compiler..." - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - echo "Compiler versions:" - $CC --version - $CXX --version - - # Build - echo "Starting chdb build with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Build start time: $(date)" - bash ./chdb/build-musl.sh - echo "Build end time: $(date)" - - # Test - echo "Running smoke test with Python 3.9..." - pyenv shell 3.9 - python --version - echo "Test start time: $(date)" - bash -x ./chdb/test_smoke.sh - echo "Test end time: $(date)" - - # Check build results - echo "Build results summary:" - ccache -s - echo "chdb directory contents:" - ls -lh chdb - echo "Build artifacts size:" - du -sh chdb - - # 6. Create and audit wheels - echo "=== Creating and auditing wheels ===" - echo "Wheel creation start time: $(date)" - echo "Available disk space before wheel build: $(df -h .)" - - # Build wheels - echo "Building wheels with Python 3.8..." - pyenv shell 3.8 - python --version - echo "Running make wheel..." - make wheel - echo "Wheel build completed at: $(date)" - echo "Initial wheel files:" - ls -lh dist/ || echo "No dist directory yet" - - # Install patchelf - echo "Installing patchelf for wheel auditing..." - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - echo "patchelf version: $(patchelf --version)" - - # Audit wheels - echo "Auditing wheels with Python 3.13..." - pyenv shell 3.13 - python --version - python -m pip install auditwheel - echo "auditwheel version: $(auditwheel --version)" - echo "Starting wheel audit at: $(date)" - auditwheel -v repair -w dist/ --plat musllinux_1_2_aarch64 dist/*.whl - echo "Wheel audit completed at: $(date)" - - # Clean up non-musllinux wheels - echo "Cleaning up non-musllinux wheels..." - echo "Before cleanup:" - ls -lh dist/ - rm -f dist/*-linux_aarch64.whl - echo "After cleanup:" - ls -lh dist/ - echo "Final wheel sizes:" - du -sh dist/* - - # 7. Test wheels - echo "=== Testing wheels ===" - echo "Wheel testing start time: $(date)" - echo "Available wheels for testing:" - ls -lh dist/*.whl - echo "Wheel file details:" - file dist/*.whl - - TOTAL_TESTS=5 - CURRENT_TEST=0 - TEST_FAILED=false - - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - CURRENT_TEST=$((CURRENT_TEST + 1)) - echo "=== Test $CURRENT_TEST/$TOTAL_TESTS: Python $version ===" - echo "Test start time: $(date)" - - echo "Switching to Python $version..." - pyenv shell $version - python --version - echo "pip version: $(python -m pip --version)" - - echo "Installing chdb wheel..." - python -m pip install dist/*.whl --force-reinstall - echo "Installation completed at: $(date)" - - echo "Running basic query test..." - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - - echo "Running full test suite..." - if make test; then - echo "Test suite PASSED for Python $version at: $(date)" - else - echo "Test suite FAILED for Python $version at: $(date)" - TEST_FAILED=true - break - fi - - pyenv shell --unset - echo "Test $CURRENT_TEST/$TOTAL_TESTS completed successfully" - echo "" - done - - echo "All wheel tests completed at: $(date)" - - # Check if any tests failed - if [ "$TEST_FAILED" = true ]; then - echo "ERROR: One or more test suites failed!" - echo "Test failure detected - aborting build process" - exit 1 - fi - - # Create test success marker file only if all tests passed - echo "All tests passed successfully!" - echo "Creating test success marker..." - touch /workspace/.test_success_marker - echo "Test success marker created at: $(date)" - - # 8. Scan chdb libraries - echo "=== Scanning chdb libraries ===" - FILES_TO_SCAN="$(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false - - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" - - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done - - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - - # Show final results - echo "=== Final wheel files ===" - ls -la ./dist/ + run: | + docker run --rm -v ${{ github.workspace }}:/workspace --privileged -e GITHUB_REF=${{ github.ref }} \ + quay.io/pypa/musllinux_1_2_aarch64 /bin/sh /workspace/.github/scripts/build-musllinux-arm64.sh continue-on-error: false # Check test success before upload - name: Verify test completion @@ -350,6 +86,9 @@ jobs: - name: Setup Python and install twine for PyPI upload run: | echo "=== Setting up Python for PyPI upload ===" + # Install pip if not available + sudo apt-get update + sudo apt-get install -y python3-pip python3 --version python3 -m pip --version echo "=== Installing twine ===" diff --git a/.github/workflows/build_musllinux_x86_wheels.yml b/.github/workflows/build_musllinux_x86_wheels.yml index 0a753c6a493..84963a104f5 100644 --- a/.github/workflows/build_musllinux_x86_wheels.yml +++ b/.github/workflows/build_musllinux_x86_wheels.yml @@ -1,306 +1,306 @@ -name: Build Linux(musllinux) x86 +# name: Build Linux(musllinux) x86 -on: - workflow_dispatch: - inputs: - TAG_NAME: - description: 'Release Version Tag' - required: true - release: - types: [created] - push: - branches: - - main - paths-ignore: - - '**/*.md' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - branches: - - main - paths-ignore: - - '**/*.md' +# on: +# workflow_dispatch: +# inputs: +# TAG_NAME: +# description: 'Release Version Tag' +# required: true +# release: +# types: [created] +# push: +# branches: +# - main +# paths-ignore: +# - '**/*.md' +# pull_request: +# types: [opened, synchronize, reopened, ready_for_review] +# branches: +# - main +# paths-ignore: +# - '**/*.md' -jobs: - build_musllinux_wheels: - name: Build musllinux wheels (Alpine Linux x86_64) - runs-on: gh-64c - if: ${{ !github.event.pull_request.draft }} - container: - image: quay.io/pypa/musllinux_1_2_x86_64 - options: --privileged - steps: - - name: Check system info - run: | - echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" - if [ -f /lib/ld-musl-x86_64.so.1 ]; then - echo "musl libc x86_64" - elif [ -f /lib/libc.musl-x86_64.so.1 ]; then - echo "musl libc x86_64" - else - echo "Not musl libc" - fi +# jobs: +# build_musllinux_wheels: +# name: Build musllinux wheels (Alpine Linux x86_64) +# runs-on: gh-64c +# if: ${{ !github.event.pull_request.draft }} +# container: +# image: quay.io/pypa/musllinux_1_2_x86_64 +# options: --privileged +# steps: +# - name: Check system info +# run: | +# echo "System: $(uname -m) $(cat /etc/os-release | grep PRETTY_NAME | cut -d'"' -f2)" +# if [ -f /lib/ld-musl-x86_64.so.1 ]; then +# echo "musl libc x86_64" +# elif [ -f /lib/libc.musl-x86_64.so.1 ]; then +# echo "musl libc x86_64" +# else +# echo "Not musl libc" +# fi - echo "=== CPU Information ===" - cat /proc/cpuinfo - echo "" - echo "=== Checking CPU requirements ===" - if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then - echo "CPU meets minimum requirements" - else - echo "CPU does not meet minimum requirements" - fi - - name: Install Python build dependencies - run: | - apk update - apk add --no-cache make build-base openssl-dev zlib-dev \ - bzip2-dev readline-dev sqlite-dev wget curl llvm \ - ncurses-dev xz-dev tk-dev libxml2-dev \ - libffi-dev linux-headers - - name: Scan SQLite vulnerabilities with grype - run: | - # Install grype and required tools - curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin +# echo "=== CPU Information ===" +# cat /proc/cpuinfo +# echo "" +# echo "=== Checking CPU requirements ===" +# if grep -q "ssse3" /proc/cpuinfo && grep -q "sse4_1" /proc/cpuinfo && grep -q "sse4_2" /proc/cpuinfo; then +# echo "CPU meets minimum requirements" +# else +# echo "CPU does not meet minimum requirements" +# fi +# - name: Install Python build dependencies +# run: | +# apk update +# apk add --no-cache make build-base openssl-dev zlib-dev \ +# bzip2-dev readline-dev sqlite-dev wget curl llvm \ +# ncurses-dev xz-dev tk-dev libxml2-dev \ +# libffi-dev linux-headers +# - name: Scan SQLite vulnerabilities with grype +# run: | +# # Install grype and required tools +# curl -sSfL https://raw.githubusercontent.com/anchore/grype/main/install.sh | sh -s -- -b /usr/local/bin - # Update grype vulnerability database - grype db update +# # Update grype vulnerability database +# grype db update - # Check SQLite vulnerabilities in installed packages - echo "Scanning SQLite packages for vulnerabilities..." - GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) - echo "Raw grype output:" - echo "$GRYPE_RAW_OUTPUT" +# # Check SQLite vulnerabilities in installed packages +# echo "Scanning SQLite packages for vulnerabilities..." +# GRYPE_RAW_OUTPUT=$(grype dir:/lib/apk/db --scope all-layers 2>/dev/null || true) +# echo "Raw grype output:" +# echo "$GRYPE_RAW_OUTPUT" - SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) +# SQLITE_SCAN_OUTPUT=$(echo "$GRYPE_RAW_OUTPUT" | grep -i sqlite || true) - if [ -n "$SQLITE_SCAN_OUTPUT" ]; then - echo "SQLite vulnerabilities found in packages! Build should be reviewed." - echo "SQLite vulnerability details:" - echo "$SQLITE_SCAN_OUTPUT" - else - echo "No SQLite vulnerabilities found" - fi - continue-on-error: false - - name: Setup pyenv - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv install 3.8:latest - pyenv install 3.9:latest - pyenv install 3.10:latest - pyenv install 3.11:latest - pyenv install 3.12:latest - pyenv install 3.13:latest - pyenv install 3.14:latest - pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 +# if [ -n "$SQLITE_SCAN_OUTPUT" ]; then +# echo "SQLite vulnerabilities found in packages! Build should be reviewed." +# echo "SQLite vulnerability details:" +# echo "$SQLITE_SCAN_OUTPUT" +# else +# echo "No SQLite vulnerabilities found" +# fi +# continue-on-error: false +# - name: Setup pyenv +# run: | +# curl https://pyenv.run | bash +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv install 3.8:latest +# pyenv install 3.9:latest +# pyenv install 3.10:latest +# pyenv install 3.11:latest +# pyenv install 3.12:latest +# pyenv install 3.13:latest +# pyenv install 3.14:latest +# pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 - # Verify installations - echo "Installed versions:" - pyenv versions - - name: Verify pyenv installations - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - echo "Verifying all required Python versions are available:" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - if ! pyenv versions --bare | grep -q "^$version"; then - echo "ERROR: Python $version is not installed!" - exit 1 - fi - echo "Python $version is installed" - done - echo "All Python versions verified successfully!" - - name: Install dependencies for all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Installing dependencies for Python $version" - pyenv shell $version - python -m pip install --upgrade pip - if [ "$version" = "3.8" ]; then - python -m pip install setuptools tox twine psutil wheel - else - python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel - fi - pyenv shell --unset - done - - name: Install clang++ for Alpine - run: | - apk add --no-cache make cmake ccache ninja yasm gawk wget - apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 - # Install Rust toolchain via rustup for proper target management - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable - source $HOME/.cargo/env - rustup toolchain install nightly-2025-07-07 - rustup component add --toolchain nightly-2025-07-07 rust-src - rustc --version - cargo --version - ccache -s - - name: Update git - run: | - apk add --no-cache git - git --version - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Configure git safe directory - run: | - git config --global --add safe.directory '*' - - name: Update submodules - run: | - git submodule update --init --recursive --jobs 4 - - name: Update version for release - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.9 +# # Verify installations +# echo "Installed versions:" +# pyenv versions +# - name: Verify pyenv installations +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# echo "Verifying all required Python versions are available:" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# if ! pyenv versions --bare | grep -q "^$version"; then +# echo "ERROR: Python $version is not installed!" +# exit 1 +# fi +# echo "Python $version is installed" +# done +# echo "All Python versions verified successfully!" +# - name: Install dependencies for all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Installing dependencies for Python $version" +# pyenv shell $version +# python -m pip install --upgrade pip +# if [ "$version" = "3.8" ]; then +# python -m pip install setuptools tox twine psutil wheel +# else +# python -m pip install setuptools tox pandas pyarrow twine psutil deltalake wheel +# fi +# pyenv shell --unset +# done +# - name: Install clang++ for Alpine +# run: | +# apk add --no-cache make cmake ccache ninja yasm gawk wget +# apk add --no-cache clang20 clang20-dev llvm20 llvm20-dev lld20 +# # Install Rust toolchain via rustup for proper target management +# curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +# source $HOME/.cargo/env +# rustup toolchain install nightly-2025-07-07 +# rustup component add --toolchain nightly-2025-07-07 rust-src +# rustc --version +# cargo --version +# ccache -s +# - name: Update git +# run: | +# apk add --no-cache git +# git --version +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - name: Configure git safe directory +# run: | +# git config --global --add safe.directory '*' +# - name: Update submodules +# run: | +# git submodule update --init --recursive --jobs 4 +# - name: Update version for release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.9 - # Install bump-my-version - python -m pip install bump-my-version - TAG_NAME=${GITHUB_REF#refs/tags/v} - bump-my-version replace --new-version $TAG_NAME - echo "Version files updated to $TAG_NAME" - pyenv shell --unset - - name: ccache - uses: hendrikmuhs/ccache-action@v1.2 - with: - key: musllinux-1-2-x86_64 - max-size: 5G - append-timestamp: true - - name: setup clang and link clang-20 to clang - run: | - ln -sf /usr/bin/clang-20 /usr/bin/clang - ln -sf /usr/bin/clang++-20 /usr/bin/clang++ - which clang++ - clang++ --version - - name: Run chdb/build-musl.sh - timeout-minutes: 600 - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - source $HOME/.cargo/env - pyenv shell 3.8 - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - bash ./chdb/build-musl.sh - pyenv shell 3.9 - bash -x ./chdb/test_smoke.sh - continue-on-error: false - - name: Scan chdb libraries with grype - run: | - echo "Scanning chdb libraries for vulnerabilities..." +# # Install bump-my-version +# python -m pip install bump-my-version +# TAG_NAME=${GITHUB_REF#refs/tags/v} +# bump-my-version replace --new-version $TAG_NAME +# echo "Version files updated to $TAG_NAME" +# pyenv shell --unset +# - name: ccache +# uses: hendrikmuhs/ccache-action@v1.2 +# with: +# key: musllinux-1-2-x86_64 +# max-size: 5G +# append-timestamp: true +# - name: setup clang and link clang-20 to clang +# run: | +# ln -sf /usr/bin/clang-20 /usr/bin/clang +# ln -sf /usr/bin/clang++-20 /usr/bin/clang++ +# which clang++ +# clang++ --version +# - name: Run chdb/build-musl.sh +# timeout-minutes: 600 +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# source $HOME/.cargo/env +# pyenv shell 3.8 +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# bash ./chdb/build-musl.sh +# pyenv shell 3.9 +# bash -x ./chdb/test_smoke.sh +# continue-on-error: false +# - name: Scan chdb libraries with grype +# run: | +# echo "Scanning chdb libraries for vulnerabilities..." - FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" +# FILES_TO_SCAN="$FILES_TO_SCAN $(find chdb/ \( -name "*.so" -o -name "*.dylib" \) 2>/dev/null || true)" - SQLITE_VULNERABILITIES_FOUND=false +# SQLITE_VULNERABILITIES_FOUND=false - for file in $FILES_TO_SCAN; do - if [ -f "$file" ]; then - echo "=== Scanning $file ===" - SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) - echo "$SCAN_OUTPUT" +# for file in $FILES_TO_SCAN; do +# if [ -f "$file" ]; then +# echo "=== Scanning $file ===" +# SCAN_OUTPUT=$(grype "$file" 2>/dev/null || true) +# echo "$SCAN_OUTPUT" - if echo "$SCAN_OUTPUT" | grep -qi sqlite; then - echo "SQLite vulnerability found in $file" - SQLITE_VULNERABILITIES_FOUND=true - fi - fi - done +# if echo "$SCAN_OUTPUT" | grep -qi sqlite; then +# echo "SQLite vulnerability found in $file" +# SQLITE_VULNERABILITIES_FOUND=true +# fi +# fi +# done - if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then - echo "SQLite vulnerabilities detected in chdb libraries!" - else - echo "No SQLite vulnerabilities found in chdb libraries" - fi - continue-on-error: false - - name: Check ccache statistics - run: | - ccache -s - ls -lh chdb - df -h - - name: Build wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ - pyenv shell 3.8 - make wheel - - name: Install patchelf from github - run: | - wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz - tar -xvf patchelf.tar.gz - cp bin/patchelf /usr/bin/ - chmod +x /usr/bin/patchelf - patchelf --version - - name: Audit wheels - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m pip install auditwheel - auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl - continue-on-error: false - - name: Show files - run: | - rm -f dist/*-linux_x86_64.whl - ls -lh dist - shell: bash - - name: Test wheel on all Python versions - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - for version in 3.9 3.10 3.11 3.12 3.13 3.14; do - echo "Testing chdb on Python $version" - pyenv shell $version - python -m pip install dist/*.whl --force-reinstall - python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" - make test - pyenv shell --unset - done - continue-on-error: false - - name: Install GitHub CLI - run: | - # Install GitHub CLI from binary release (Alpine Linux compatible) - wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_amd64.tar.gz -O gh.tar.gz - tar -xf gh.tar.gz - cp gh_*/bin/gh /usr/local/bin/ - chmod +x /usr/local/bin/gh - if ! gh --version; then - echo "ERROR: GitHub CLI installation failed!" - exit 1 - fi - echo "GitHub CLI installed successfully" - - name: Upload wheels to release - if: startsWith(github.ref, 'refs/tags/v') - run: | - gh release upload ${{ github.ref_name }} dist/*.whl --clobber - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - uses: actions/upload-artifact@v4 - with: - name: chdb-artifacts-musllinux-x86_64 - path: | - ./dist/*.whl - overwrite: true - - name: Upload pypi - if: startsWith(github.ref, 'refs/tags/v') - run: | - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - pyenv shell 3.13 - python -m twine upload dist/*.whl - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - - name: Keep killall ccache and wait for ccache to finish - if: always() - run: | - sleep 60 - while ps -ef | grep ccache | grep -v grep; do \ - killall ccache; \ - sleep 10; \ - done +# if [ "$SQLITE_VULNERABILITIES_FOUND" = true ]; then +# echo "SQLite vulnerabilities detected in chdb libraries!" +# else +# echo "No SQLite vulnerabilities found in chdb libraries" +# fi +# continue-on-error: false +# - name: Check ccache statistics +# run: | +# ccache -s +# ls -lh chdb +# df -h +# - name: Build wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# export CC=/usr/bin/clang +# export CXX=/usr/bin/clang++ +# pyenv shell 3.8 +# make wheel +# - name: Install patchelf from github +# run: | +# wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-x86_64.tar.gz -O patchelf.tar.gz +# tar -xvf patchelf.tar.gz +# cp bin/patchelf /usr/bin/ +# chmod +x /usr/bin/patchelf +# patchelf --version +# - name: Audit wheels +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.13 +# python -m pip install auditwheel +# auditwheel -v repair -w dist/ --plat musllinux_1_2_x86_64 dist/*.whl +# continue-on-error: false +# - name: Show files +# run: | +# rm -f dist/*-linux_x86_64.whl +# ls -lh dist +# shell: bash +# - name: Test wheel on all Python versions +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# for version in 3.9 3.10 3.11 3.12 3.13 3.14; do +# echo "Testing chdb on Python $version" +# pyenv shell $version +# python -m pip install dist/*.whl --force-reinstall +# python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" +# make test +# pyenv shell --unset +# done +# continue-on-error: false +# - name: Install GitHub CLI +# run: | +# # Install GitHub CLI from binary release (Alpine Linux compatible) +# wget https://github.com/cli/cli/releases/download/v2.82.1/gh_2.82.1_linux_amd64.tar.gz -O gh.tar.gz +# tar -xf gh.tar.gz +# cp gh_*/bin/gh /usr/local/bin/ +# chmod +x /usr/local/bin/gh +# if ! gh --version; then +# echo "ERROR: GitHub CLI installation failed!" +# exit 1 +# fi +# echo "GitHub CLI installed successfully" +# - name: Upload wheels to release +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# gh release upload ${{ github.ref_name }} dist/*.whl --clobber +# env: +# GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} +# - uses: actions/upload-artifact@v4 +# with: +# name: chdb-artifacts-musllinux-x86_64 +# path: | +# ./dist/*.whl +# overwrite: true +# - name: Upload pypi +# if: startsWith(github.ref, 'refs/tags/v') +# run: | +# export PATH="$HOME/.pyenv/bin:$PATH" +# eval "$(pyenv init -)" +# pyenv shell 3.13 +# python -m twine upload dist/*.whl +# env: +# TWINE_USERNAME: __token__ +# TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} +# - name: Keep killall ccache and wait for ccache to finish +# if: always() +# run: | +# sleep 60 +# while ps -ef | grep ccache | grep -v grep; do \ +# killall ccache; \ +# sleep 10; \ +# done \ No newline at end of file diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh index 33066b916fe..6095f4747e5 100755 --- a/chdb/build_pybind11.sh +++ b/chdb/build_pybind11.sh @@ -33,8 +33,33 @@ build_pybind11_nonlimitedapi() { local py_version=$1 echo "Building pybind11 nonlimitedapi library for Python ${py_version}..." + local full_py_version="" + local custom_python_path="" + + if command -v pyenv >/dev/null 2>&1; then + full_py_version=$(pyenv versions --bare | grep "^${py_version}\." | head -n 1) + + if [ -n "$full_py_version" ]; then + custom_python_path="$HOME/.pyenv/versions/${full_py_version}/bin/python3" + if [ -f "$custom_python_path" ]; then + echo "Found pyenv Python ${full_py_version} at: $custom_python_path" + else + echo "Warning: pyenv Python ${full_py_version} not found at expected path" + custom_python_path="" + fi + else + echo "Warning: Python ${py_version}.x not found in pyenv versions" + fi + else + echo "Warning: pyenv not found" + fi + local py_cmake_args="${CMAKE_ARGS} -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version}" + if [ -n "$custom_python_path" ]; then + py_cmake_args="${py_cmake_args} -DCHDB_CUSTOM_PYTHON_EXECUTABLE=${custom_python_path}" + fi + cmake ${py_cmake_args} -DENABLE_PYTHON=1 .. # Build only the pybind11 targets diff --git a/contrib/pybind11-cmake/CMakeLists.txt b/contrib/pybind11-cmake/CMakeLists.txt index 0427cf47b71..3a34a169049 100644 --- a/contrib/pybind11-cmake/CMakeLists.txt +++ b/contrib/pybind11-cmake/CMakeLists.txt @@ -6,6 +6,13 @@ endif() string(REPLACE "-Wl,-z,defs" "" CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS}") +# Use custom Python path if provided +if (DEFINED CHDB_CUSTOM_PYTHON_EXECUTABLE) + set(Python_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + set(Python3_EXECUTABLE "${CHDB_CUSTOM_PYTHON_EXECUTABLE}") + message(STATUS "Using custom Python executable: ${CHDB_CUSTOM_PYTHON_EXECUTABLE}") +endif() + if (DEFINED PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION) find_package(Python ${PYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION} EXACT REQUIRED COMPONENTS Interpreter Development) else()