Merge pull request #385 from wudidapaopao/support_arrow_scan

Author: auxten

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -217,6 +217,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Check ccache statistics
         run: |
           ccache -s

.github/workflows/build_linux_x86_wheels.yml

@@ -217,6 +217,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Check ccache statistics
         run: |
           ccache -s

.github/workflows/build_macos_arm64_wheels.yml

@@ -24,6 +24,19 @@ jobs:
     name: Build Universal Wheel (macOS ARM64)
     runs-on: macos-14-xlarge
     steps:
+      - name: Check machine architecture
+        run: |
+          echo "=== Machine Architecture Information ==="
+          echo "Machine type: $(uname -m)"
+          echo "Architecture: $(arch)"
+          echo "System info: $(uname -a)"
+          echo "Hardware info:"
+          system_profiler SPHardwareDataType | grep "Chip\|Processor"
+          if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then
+            echo "This is an ARM64 (Apple Silicon) machine"
+          else
+            echo "This is an x86_64 (Intel) machine"
+          fi
       - name: Free up disk space (Initial)
         run: |
           # Clean Homebrew cache
@@ -201,6 +214,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Build wheels
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"

.github/workflows/build_macos_x86_wheels.yml

@@ -25,6 +25,19 @@ jobs:
     runs-on: macos-15-intel
     timeout-minutes: 600
     steps:
+      - name: Check machine architecture
+        run: |
+          echo "=== Machine Architecture Information ==="
+          echo "Machine type: $(uname -m)"
+          echo "Architecture: $(arch)"
+          echo "System info: $(uname -a)"
+          echo "Hardware info:"
+          system_profiler SPHardwareDataType | grep "Chip\|Processor"
+          if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then
+            echo "This is an ARM64 (Apple Silicon) machine"
+          else
+            echo "This is an x86_64 (Intel) machine"
+          fi
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -202,6 +215,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Build wheels
         run: |
           export PATH="$HOME/.pyenv/bin:$PATH"

README.md

@@ -416,11 +416,7 @@ chDB automatically converts Python dictionary objects to ClickHouse JSON types f
       ```
     - Columns are converted to `String` if sampling finds non-dictionary values.
 
-2. **Arrow Table**
-    - `struct` type columns are automatically mapped to JSON columns.
-    - Nested structures preserve type information.
-
-3. **chdb.PyReader**
+2. **chdb.PyReader**
     - Implement custom schema mapping in `get_schema()`:
       ```python
       def get_schema(self):

chdb/__init__.py

@@ -103,7 +103,9 @@ def to_arrowTable(res):
         raise ImportError("Failed to import pyarrow or pandas") from None
     if len(res) == 0:
         return pa.Table.from_batches([], schema=pa.schema([]))
-    return pa.RecordBatchFileReader(res.bytes()).read_all()
+
+    memview = res.get_memview()
+    return pa.RecordBatchFileReader(memview.view()).read_all()
 
 
 # return pandas dataframe

chdb/state/sqlitelike.py

@@ -62,7 +62,9 @@ def to_arrowTable(res):
         raise ImportError("Failed to import pyarrow or pandas") from None
     if len(res) == 0:
         return pa.Table.from_batches([], schema=pa.schema([]))
-    return pa.RecordBatchFileReader(res.bytes()).read_all()
+
+    memview = res.get_memview()
+    return pa.RecordBatchFileReader(memview.view()).read_all()
 
 
 # return pandas dataframe