Merge branch 'main' into support_arrow_scan

wudidapaopao · web-flow · commit 4894866fb30f · 2025-10-21T11:33:20.000+02:00
diff --git a/.github/workflows/build_macos_arm64_wheels.yml b/.github/workflows/build_macos_arm64_wheels.yml
@@ -22,7 +22,7 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (macOS ARM64)
-    runs-on: macos-13-xlarge
+    runs-on: macos-14-xlarge
     steps:
       - name: Check machine architecture
         run: |
@@ -37,6 +37,17 @@ jobs:
           else
             echo "This is an x86_64 (Intel) machine"
           fi
+      - name: Free up disk space (Initial)
+        run: |
+          # Clean Homebrew cache
+          brew cleanup -s 2>/dev/null || true
+          rm -rf "$(brew --cache)" 2>/dev/null || true
+          sudo rm -rf ~/Library/Developer/Xcode/DerivedData 2>/dev/null || true
+          sudo rm -rf ~/Library/Caches/com.apple.dt.Xcode 2>/dev/null || true
+          sudo rm -rf /Users/runner/Library/Android 2>/dev/null || true
+          sudo rm -rf /tmp/* 2>/dev/null || true
+          echo "=== Disk usage after cleanup ==="
+          df -h
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -93,6 +104,7 @@ jobs:
           brew install openssl@3 || echo "OpenSSL install failed, continuing..."
           brew install --ignore-dependencies llvm@19
           brew install git ninja libtool gettext binutils grep findutils nasm lld@19 libiconv
+          brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
           export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH
@@ -115,6 +127,23 @@ jobs:
       - name: Update submodules
         run: |
           git submodule update --init --recursive --jobs 4
+      - name: Free up disk space (Before compilation)
+        run: |
+          echo "=== Disk usage before compilation cleanup ==="
+          df -h
+          brew cleanup -s 2>/dev/null || true
+          rm -rf "$(brew --cache)" 2>/dev/null || true
+          rm -rf ~/.cache/pip 2>/dev/null || true
+          rm -rf ~/.pyenv/.cache 2>/dev/null || true
+          rm -rf ~/.cargo/registry/cache 2>/dev/null || true
+          echo "=== Disk usage after cleanup ==="
+          df -h
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ${{ runner.os }}-arm64-ccache
+          max-size: 5G
+          append-timestamp: true
       - name: Run chdb/build.sh
         timeout-minutes: 600
         run: |
@@ -207,16 +236,23 @@ jobs:
           sudo sysctl kern.corefile=$PWD/tmp/core/core.%P
           sudo sysctl kern.coredump=1
           ulimit -c unlimited
+      - name: Free up disk space
+        run: |
+          # Clean more build artifacts
+          rm -rf buildlib/contrib 2>/dev/null || true
+          rm -rf buildlib/base 2>/dev/null || true
+          rm -rf buildlib/src 2>/dev/null || true
       - name: Test wheel on all Python versions
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
             echo "Testing chdb on Python $version"
             pyenv shell $version
-            python -m pip install dist/*.whl --force-reinstall
+            python -m pip install dist/*.whl --force-reinstall --no-cache-dir
             python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')"
             make test
+            python -m pip uninstall -y chdb
             pyenv shell --unset
           done
         continue-on-error: false
diff --git a/.github/workflows/build_macos_x86_wheels.yml b/.github/workflows/build_macos_x86_wheels.yml
@@ -22,7 +22,8 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (macOS x86_64)
-    runs-on: macos-14-large
+    runs-on: macos-15-intel
+    timeout-minutes: 600
     steps:
       - name: Check machine architecture
         run: |
@@ -43,13 +44,12 @@ jobs:
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
 
-          pyenv install 3.8:latest
           pyenv install 3.9:latest
           pyenv install 3.10:latest
           pyenv install 3.11:latest
           pyenv install 3.12:latest
           pyenv install 3.13:latest
-          pyenv global 3.8 3.9 3.10 3.11 3.12 3.13
+          pyenv global 3.9 3.10 3.11 3.12 3.13
 
           echo "Installed versions:"
           pyenv versions
@@ -61,7 +61,7 @@ jobs:
           pyenv versions
           echo ""
           echo "Verifying all required Python versions are available:"
-          for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
+          for version in 3.9 3.10 3.11 3.12 3.13; do
             if ! pyenv versions --bare | grep -q "^$version"; then
               echo "ERROR: Python $version is not installed!"
               exit 1
@@ -73,7 +73,7 @@ jobs:
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
+          for version in 3.9 3.10 3.11 3.12 3.13; do
             echo "Installing dependencies for Python $version"
             pyenv shell $version
             python -m pip install --upgrade pip
@@ -135,30 +135,39 @@ jobs:
       - name: Update submodules
         run: |
           git submodule update --init --recursive --jobs 4
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2
+        with:
+          key: ${{ runner.os }}-x86_64-ccache
+          max-size: 10G
+          append-timestamp: true
+        env:
+          CCACHE_NOHASHDIR: "true"
       - name: Run chdb/build.sh
         timeout-minutes: 600
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
           source ~/.cargo/env
-          pyenv shell 3.8
+          pyenv shell 3.9
           export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
           bash gen_manifest.sh
           bash ./chdb/build.sh
-          pyenv shell 3.8
+          pyenv shell 3.9
           bash -x ./chdb/test_smoke.sh
         continue-on-error: false
       - name: Run chdb/build/build_static_lib.sh
         timeout-minutes: 600
         run: |
-          export PATH=$HOME/.pyenv/bin:$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
+          export PATH="$HOME/.pyenv/bin:$PATH"
+          eval "$(pyenv init -)"
+          source ~/.cargo/env
+          export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
-          source ~/.cargo/env
-          eval "$(pyenv init -)"
-          pyenv shell 3.8
+          pyenv shell 3.9
           bash ./chdb/build/build_static_lib.sh
           pyenv shell --unset
         continue-on-error: false
@@ -215,25 +224,31 @@ jobs:
           export PATH=$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:/usr/local/opt/grep/libexec/gnubin:/usr/local/opt/binutils/bin:$PATH:/usr/local/opt/findutils/libexec/gnubin
           export CC=$(brew --prefix llvm@19)/bin/clang
           export CXX=$(brew --prefix llvm@19)/bin/clang++
-          pyenv shell 3.8
+          pyenv shell 3.9
           make wheel
       - name: Fix wheel platform tag
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          pyenv shell 3.8
+          pyenv shell 3.9
           python -m wheel tags --platform-tag=macosx_10_15_x86_64 --remove dist/*.whl
       - name: Setup core dump collection
         run: |
           mkdir -p tmp/core
           sudo sysctl kern.corefile=$PWD/tmp/core/core.%P
           sudo sysctl kern.coredump=1
           ulimit -c unlimited
+      - name: Free up disk space
+        run: |
+          # Clean more build artifacts
+          rm -rf buildlib/contrib 2>/dev/null || true
+          rm -rf buildlib/base 2>/dev/null || true
+          rm -rf buildlib/src 2>/dev/null || true
       - name: Test wheel on all Python versions
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          for version in 3.8 3.9 3.10 3.11 3.12 3.13; do
+          for version in 3.9 3.10 3.11 3.12 3.13; do
             echo "Testing chdb on Python $version"
             pyenv shell $version
             python -m pip install dist/*.whl --force-reinstall
@@ -305,7 +320,7 @@ jobs:
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"
           eval "$(pyenv init -)"
-          pyenv shell 3.8
+          pyenv shell 3.9
           python -m twine upload dist/*.whl
         env:
           TWINE_USERNAME: __token__
diff --git a/chdb/build.sh b/chdb/build.sh
@@ -30,7 +30,6 @@ if [ "$(uname)" == "Darwin" ]; then
     # if Darwin ARM64 (M1, M2), disable AVX
     if [ "$(uname -m)" == "arm64" ]; then
         CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
-        COMPILER_CACHE="-DCOMPILER_CACHE=disabled"
         LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
     else
         LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
@@ -110,7 +109,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0
     ${CPU_FEATURES} \
     -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \
     -DENABLE_LIBFIU=1 \
-    ${COMPILER_CACHE} \
     -DCHDB_VERSION=${CHDB_VERSION} \
     "
 
@@ -239,6 +237,10 @@ ls -lh ${LIBCHDB}
 
 # build chdb python module
 py_version="3.8"
+# check current os type and architecture for py_version
+if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then
+    py_version="3.9"
+fi
 current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
 if [ "$current_py_version" != "$py_version" ]; then
     echo "Error: Current Python version is $current_py_version, but required version is $py_version"
diff --git a/chdb/build/build_static_lib.sh b/chdb/build/build_static_lib.sh
@@ -25,9 +25,6 @@ if [ "$(uname)" == "Darwin" ]; then
     MYSQL="-DENABLE_MYSQL=0"
     ICU="-DENABLE_ICU=0"
     CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
-    if [ "$(uname -m)" == "arm64" ]; then
-        COMPILER_CACHE="-DCOMPILER_CACHE=disabled"
-    fi
 elif [ "$(uname)" == "Linux" ]; then
     GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1"
     UNWIND="-DUSE_UNWIND=1"
@@ -69,7 +66,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0
     ${CPU_FEATURES} \
     -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \
     -DENABLE_LIBFIU=1 \
-    ${COMPILER_CACHE} \
     -DCHDB_VERSION=${CHDB_VERSION} \
     "
 
diff --git a/chdb/build_pybind11.sh b/chdb/build_pybind11.sh
@@ -64,6 +64,11 @@ build_pybind11_nonlimitedapi() {
 
 build_all_pybind11_nonlimitedapi() {
     local python_versions=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13")
+    
+    # Skip Python 3.8 for macOS x86_64
+    if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then
+        python_versions=("3.9" "3.10" "3.11" "3.12" "3.13")
+    fi
 
     echo "Building pybind11 nonlimitedapi libraries for all Python versions..."
 
