paultiq · paultiq · Oct 2, 2025 · Oct 2, 2025 · Oct 2, 2025 · Oct 2, 2025
diff --git a/.github/workflows/dev_testing.yml b/.github/workflows/dev_testing.yml
@@ -0,0 +1,225 @@
+name: On Push Fast Tests
+on: push
+
+# This workflow is runnable locally for debugging:
+# using https://github.com/nektos/act
+# 
+jobs:
+  build_wheels:
+    name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.os }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    strategy:
+      fail-fast: false
+      matrix:
+        python: [ cp39, cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314t, 
+        platform:
+          #- { os: windows-2025,     arch: amd64,      cibw_system: win }
+          - { os: ubuntu-latest,     arch: x86_64,     cibw_system: manylinux }
+          # - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          #- { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          #- { os: macos-26,         arch: arm64,      cibw_system: macosx }
+          # - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          # - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+        #exclude:
+        #  - { python: cp314t, platform: { os: windows-2025 } }  
+
+    runs-on: ${{ matrix.platform.os }}
+    env:
+      CIBW_TEST_SOURCES: tests
+      CIBW_BEFORE_TEST_LINUX: >
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+        uv pip install -r pylock.toml
+      CIBW_BEFORE_TEST_MACOS: >
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+        uv pip install -r pylock.toml
+      CIBW_BEFORE_TEST_WINDOWS: >
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+        uv pip install -r pylock.toml
+      CIBW_TEST_COMMAND: >
+        uv run -v pytest ./tests/fast --verbose --ignore=./tests/stubs --durations=5 -n 2
+      #  && uv run -v pytest ./tests/fast/threading --durations=5 --parallel-threads=10 --iterations=5 -n 2
+
+      CMAKE_VERBOSE_MAKEFILE: "ON"
+
+      SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}
+      SCCACHE_BASEDIR: "/tmp/duckdb-build"
+      ACTIONS_CACHE_SERVICE_V2: "1"
+
+      TMPDIR: "/tmp/duckdb-build"
+      TEMP: "/tmp/duckdb-build"  
+    steps:
+      - name: Set SCCACHE_GHA_ENABLED
+        run: echo "SCCACHE_GHA_ENABLED=${{ env.ACT == 'true' && 'off' || 'on' }}" >> $GITHUB_ENV
+      - name: Checkout DuckDB Python
+        if: ${{ env.ACT != 'true' }}
+        uses: actions/checkout@v5
+        with:
+          # ref: ${{ inputs.duckdb-python-sha }}
+          fetch-depth: 0
+          submodules: true
+      - name: Checkout DuckDB
+        if: ${{ env.ACT != 'true' && inputs['duckdb-sha'] != '' }}
+        shell: bash
+        continue-on-error: true  # sometimes helpful when merging branch->branch
+        run: |
+          cd external/duckdb
+          git fetch origin
+          git checkout ${{ inputs.duckdb-sha }}
+      - name: Configure Cache Env
+        # if: ${{ env.ACT != 'true' }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+      # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
+      - name: Set OVERRIDE_GIT_DESCRIBE
+        shell: bash
+        if: ${{ inputs.set-version != '' }}
+        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
+
+      # # Install Astral UV, which will be used as build-frontend for cibuildwheel
+      - uses: astral-sh/setup-uv@v6
+        if: ${{ env.ACT != 'true' }}
+        with:
+          # version: "0.8.19"  # Version is important - it determines which Python build you get. Bump when new version of Python releases. 
+          enable-cache: false
+          activate-environment: true
+          cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+          python-version: ${{ matrix.python }}
+
+      # Used when running workflows locally
+      - name: Install UV
+        if: ${{ env.ACT == 'true' }}
+        shell: bash
+        env:
+          TMPDIR: /tmp
+        run: |
+          curl -LsSf https://astral.sh/uv/install.sh | sh
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+          export PATH="$HOME/.local/bin:$PATH"
+          uv --version
+
+      - name: Install sccache
+        if: ${{ matrix.platform.cibw_system == 'win' }}
+        shell: bash
+        # TODO: Find a better PATH dir to put this in
+        run: |
+          curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-pc-windows-msvc.zip -o sccache.zip
+          unzip sccache.zip
+          cp sccache-*/sccache.exe "C:/Strawberry/c/bin/sccache.exe"
+          sccache --version
+
+      # Load MSVC environment, needed for Windows - ninja builds
+      - uses: ilammy/msvc-dev-cmd@v1
+        if: ${{ matrix.platform.cibw_system == 'win' }}
+        with:
+          arch: ${{ matrix.platform.arch }}
+
+      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
+        uses: pypa/[email protected]
+        env:
+          CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+
+          #CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation"
+          UV_PROJECT_ENVIRONMENT: /project/.venv
+          #UV_NO_BUILD_ISOLATION: "true"
+          CMAKE_C_COMPILER_LAUNCHER: ""
+          CMAKE_CXX_COMPILER_LAUNCHER: ""
+          # CFLAGS: "-Wno-attributes" 
+          # CXXFLAGS: "-Wno-attributes"
+          # UV_PYTHON: ${{ matrix.python }}
+          PYTHON_GIL: 1
+          CIBW_BUILD_VERBOSITY: 3
+          # SCCACHE Notes:
+          # - Without the /project dir and tmpdirs (not sure exactly which mattered), the python environment is installed to a random tmp dir, which breaks
+          # the cache key... so, only the external/duckdb would cache, and not the project itself
+          # - GHA cache is limited to 10GB LRU.
+          # - SCCACHE_BASEDIR is not implemented in sccache (https://github.com/mozilla/sccache/issues/35)
+          # - Using -fdebug-prefix-map/-fmacro-prefix-map for path normalization instead (Mozilla Firefox approach)
+          # - Mozilla bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1524662 "Support gcc/clang-like build path prefix map"
+          # - UV creates random build-env-* dirs causing cache misses (https://github.com/astral-sh/uv/issues/13096)
+          # - Using --no-isolation to eliminate random UV build environments
+          #
+          # SCCACHE_LOG: trace
+          # SCCACHE_LOG_LEVEL: trace
+          # RUST_LOG: trace
+          # SCCACHE_NO_DAEMON: "1"
+          # PYTHONPATH="/project" ("." on mac and windows)
+
+          # UV_NO_BUILD_ISOLATION="true"
+          # no-build-isolation uses the same build-env path for each build, stable paths needed for caching
+          # SCCACHE_BASEDIR="/tmp/duckdb-build"
+          # TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build"
+          # UV_PROJECT_ENVIRONMENT="/project/.venv"
+
+          #SCCACHE_BASEDIR="/tmp/duckdb-build"
+          #TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build"
+          #UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/.venv"
+
+          # SCCACHE_BASEDIR="/tmp/duckdb-build"
+          # TMPDIR="C:/tmp/duckdb-build" TEMP="C:/tmp/duckdb-build"
+          # UV_PROJECT_ENVIRONMENT="C:/project/.venv"
+          #CMAKE_BUILD_TYPE=RelWithDebInfo
+          CIBW_ENVIRONMENT_WINDOWS: >
+            SCCACHE_GHA_ENABLED=true
+            CMAKE_BUILD_TYPE=Release
+            PYTHONPATH="."
+            TMPDIR="C:/tmp/duckdb-build"
+            TEMP="C:/tmp/duckdb-build"
+            CMAKE_ARGS="-DPython3_EXECUTABLE=C:/tmp/duckdb-build/stable-venv/Scripts/python.exe -DPython_EXECUTABLE=C:/tmp/duckdb-build/stable-venv/Scripts/python.exe -DCMAKE_PREFIX_PATH=C:/tmp/duckdb-build/stable-venv"
+          CIBW_ENVIRONMENT_MACOS: >
+            SCCACHE_GHA_ENABLED=true
+            PYTHONPATH="."
+            TMPDIR="/tmp/duckdb-build"
+            TEMP="/tmp/duckdb-build"
+            UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/.venv"
+            CIBW_BUILD_DIR="/tmp/duckdb-build/build"
+            PATH="/tmp/duckdb-build/stable-venv/bin:$PATH"
+            CMAKE_ARGS="-DPython3_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DPython_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DCMAKE_PREFIX_PATH=/tmp/duckdb-build/stable-venv"
+          CIBW_ENVIRONMENT_LINUX: >
+            SCCACHE_GHA_ENABLED=true
+            PYTHONPATH="/project"
+            TMPDIR="/tmp/duckdb-build"
+            TEMP="/tmp/duckdb-build"
+            UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/stable-venv"
+            CMAKE_ARGS="-DPython3_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DPython_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DCMAKE_PREFIX_PATH=/tmp/duckdb-build/stable-venv"
+
+          CIBW_ENVIRONMENT_PASS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER SCCACHE_LOG SCCACHE_LOG_LEVEL SCCACHE_NO_DAEMON RUST_LOG SCCACHE_BASEDIR TMPDIR TEMP PIP_CACHE_DIR UV_PROJECT_ENVIRONMENT PYTHONPATH UV_NO_BUILD_ISOLATION UV_CACHE_DIR UV_PYTHON CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER CMAKE_VERBOSE_MAKEFILE CIBW_BUILD_FRONTEND PYTHON_GIL CIBW_BUILD_VERBOSITY CIBW_BUILD_DIR VIRTUAL_ENV CMAKE_ARGS
+
+          CIBW_BEFORE_BUILD_LINUX: >
+            if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi &&
+            curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz &&
+            cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin &&
+            sccache --show-stats &&
+            mkdir -p /tmp/duckdb-build /tmp/pip-cache &&
+            uv venv /tmp/duckdb-build/stable-venv --python "$(which python)" &&
+            uv pip install --python /tmp/duckdb-build/stable-venv/bin/python 'pybind11[global]>=2.6.0'
+          CIBW_BEFORE_BUILD_MACOS: >
+            mkdir -p /tmp/duckdb-build /tmp/duckdb-build/uv-cache /tmp/pip-cache &&
+            brew install sccache &&
+            rm -f /tmp/duckdb-build/stable-venv &&
+            ln -sf "$VIRTUAL_ENV" /tmp/duckdb-build/stable-venv &&
+            uv pip install scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10'
+
+          # Build dependencies installs needed if using no-isolation
+          # uv pip install scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10' &&
+          CIBW_BEFORE_BUILD_WINDOWS: >
+            (mkdir "C:\tmp\duckdb-build" 2>nul & mkdir "C:\tmp\duckdb-build\uv-cache" 2>nul & mkdir "C:\tmp\pip-cache" 2>nul) &&
+            del "C:\Strawberry\c\bin\ccache.exe" 2>nul &&
+            sccache --show-stats &&
+            mklink /j "C:\tmp\duckdb-build\stable-venv" "%VIRTUAL_ENV%" &&
+            call "C:\tmp\duckdb-build\stable-venv\Scripts\activate.bat" &&
+            uv pip install "pybind11[global]>=2.6.0"
+
+          # &&
+          # uv pip install scikit-build-core "pybind11[global]>=2.6.0" setuptools-scm "cmake>=3.29.0" "ninja>=1.10"
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.python }}-${{ matrix.platform.os }}_${{ matrix.platform.arch }}
+          path: wheelhouse/*.whl
+          compression-level: 0
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -46,6 +46,10 @@ else()
 endif()
 duckdb_add_library(duckdb_target)
 
+# Use unity for the external duckdb_target
+set_target_properties(duckdb_target PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 32)
+
+
 # Bundle in INTERFACE library
 add_library(_duckdb_dependencies INTERFACE)
 target_link_libraries(_duckdb_dependencies INTERFACE
@@ -77,6 +81,9 @@ pybind11_add_module(_duckdb
         $<TARGET_OBJECTS:python_relation>
         $<TARGET_OBJECTS:python_type>
 )
+# don't use unity for the duckdb-python code - we just want one file per target
+set_target_properties(_duckdb PROPERTIES UNITY_BUILD OFF) 
+
 # add _duckdb_dependencies
 target_link_libraries(_duckdb PRIVATE _duckdb_dependencies)
 

diff --git a/pyproject.toml b/pyproject.toml
@@ -46,7 +46,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l
     "ipython", # used in duckdb.query_graph
     "fsspec",  # used in duckdb.filesystem
     "numpy",   # used in duckdb.experimental.spark and in duckdb.fetchnumpy()
-    "pandas; python_version < '3.14'",  # used for pandas dataframes all over the place
+    "pandas",  # used for pandas dataframes all over the place
     "pyarrow; python_version < '3.14'", # used for pyarrow support
     "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package)
 ]
@@ -73,6 +73,7 @@ cmake.version = ">=3.29.0"
 ninja.version = ">=1.10"
 ninja.make-fallback = false
 metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
+#build.tool-args = ["-v", "-d", "stats"]
 
 [tool.scikit-build.wheel]
 cmake = true
@@ -123,6 +124,39 @@ if.env.COVERAGE = false
 inherit.cmake.define = "append"
 cmake.define.DISABLE_UNITY = "1"
 
+[[tool.scikit-build.overrides]]
+if.platform-system = "^win32"
+inherit.cmake.define = "append"
+
+# Verbose Makefile breaks sccacheing
+cmake.define.CMAKE_VERBOSE_MAKEFILE = "OFF" 
+cmake.define.CMAKE_C_FLAGS_INIT="/Z7"
+cmake.define.CMAKE_CXX_FLAGS_INIT="/Z7"
+cmake.define.CMAKE_MSVC_DEBUG_INFORMATION_FORMAT="Embedded"
+#build.tool-args = ["/m:4"]
+cmake.args = [
+    "-G", "Ninja",
+    "--log-level=DEBUG",
+    "-DCMAKE_BUILD_TYPE=Release"
+]
+
+[[tool.scikit-build.overrides]]
+# Windows Free-Threading
+if.platform-system = "^win32"
+if.abi-flags = "t"
+inherit.cmake.define = "append"
+# cmake.build-type = "RelWithDebInfo"
+# /EHsc: Solution for many of the crashes observed on windows
+# /Z7 / Embedded: Needed for sccache
+cmake.define.CMAKE_MSVC_DEBUG_INFORMATION_FORMAT="Embedded"
+cmake.define.CMAKE_C_FLAGS_INIT="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED"
+cmake.define.CMAKE_CXX_FLAGS_INIT="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED"
+
+#build.tool-args = ["/m:4"]
+#, "/verbosity:diagnostic", "/property:UseCommandLineEcho=true"]
+
+# for msbuild
+
 [tool.scikit-build.sdist]
 include = [
     "README.md",
@@ -219,14 +253,16 @@ torchvision = [ { index = "pytorch-cpu" } ]
 [dependency-groups] # used for development only, requires pip >=25.1.0
 stubdeps = [ # dependencies used for typehints in the stubs
     "fsspec",
-    "pandas; python_version < '3.14'",
+    "pandas",
     "polars; python_version < '3.14'",
     "pyarrow; python_version < '3.14'",
 ]
 test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    "pytest-xdist", #  multi-processed tests, if `-n <num_workers> | auto`
+    "pytest-randomly", # randomizes test order to ensure no test dependencies, enabled on install
     "mypy",
     "coverage",
     "gcovr; python_version < '3.14'",
@@ -241,7 +277,7 @@ test = [ # dependencies used for running tests
     "requests",
     "urllib3",
     "fsspec>=2022.11.0",
-    "pandas>=2.0.0; python_version < '3.14'",
+    "pandas>=2.0.0",
     "pyarrow>=18.0.0; python_version < '3.14'",
     "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')",
     "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'",
@@ -256,7 +292,7 @@ scripts = [ # dependencies used for running scripts
     "ipython",
     "ipywidgets",
     "numpy",
-    "pandas; python_version < '3.14'",
+    "pandas",
     "pcpp",
     "polars; python_version < '3.14'",
     "pyarrow; python_version < '3.14'",
@@ -306,6 +342,7 @@ filterwarnings = [
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning",
 ]
+timeout = 600  # don't let individual tests "hang"
 
 [tool.coverage.run]
 branch = true
@@ -388,4 +425,5 @@ before-build = ["yum install -y ccache"]
 before-build = ["brew install ccache"]
 
 [tool.cibuildwheel.windows]
-before-build = ["choco install ccache"]
+# Strawberry ccache is already installed on the cibuildwheel images
+before-build = []
diff --git a/util/localbuild.sh b/util/localbuild.sh
@@ -0,0 +1,33 @@
+#export UV_PYTHON=3.14.0rc3+freethreaded
+export UV_PYTHON=3.12
+export UV_PROJECT_ENVIRONMENT=.venv_$UV_PYTHON
+
+if [ ! -d $UV_PROJECT_ENVIRONMENT ]; then
+    uv venv
+else
+    echo $UV_PROJECT_ENVIRONMENT exists
+fi 
+
+source $UV_PROJECT_ENVIRONMENT/bin/activate
+uv sync --no-install-project
+
+export CCACHE_PROGRAM=ccache
+export PYTHON_GIL=1
+# export UV_CACHE_DIR="$(pwd)/.cache"
+export CCACHE_BASEDIR=/home/ec2-user/git/duckdb-pythonf
+#export CCACHE_NOHASHDIR=1
+#export UV_NO_BUILD_ISOLATION=true
+#export UV_NO_EDITABLE=true
+#export SKBUILD_EDITABLE_MODE=redirect
+export UV_BUILD_DIR=/home/ec2-user/git/duckdb-pythonf/build
+export CMAKE_BUILD_DIR="/home/ec2-user/git/duckdb-pythonf/build"
+export SKBUILD_BUILD_DIR="/home/ec2-user/git/duckdb-pythonf/build"
+export SKBUILD_CMAKE_ARGS="-B/home/ec2-user/git/duckdb-pythonf/build"
+
+
+export CMAKE_ARGS="-DPython3_EXECUTABLE=$UV_PROJECT_ENVIRONMENT/bin/python -DPython_EXECUTABLE=$UV_PROJECT_ENVIRONMENT/bin/python -DCMAKE_PREFIX_PATH=$UV_PROJECT_ENVIRONMENT"
+
+uv sync --no-build-isolation -vv  --reinstall
+
+# uv sync --no-build-isolation --reinstall -vv --no-editable 
+