SCCACHE Builds

.github/workflows/on_pr.yml

@@ -2,8 +2,7 @@ name: Tests and builds on PR
 on:
   pull_request:
     branches:
-      - main
-      - v*.*-*
+      - '**'
     types: [opened, reopened, ready_for_review, converted_to_draft, synchronize]
     paths-ignore:
       - '**.md'
@@ -13,10 +12,11 @@ on:
       - '.github//**'
       - '!.github/workflows/on_push.yml'
       - '!.github/workflows/coverage.yml'
+    workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
   submodule_sanity_guard:
@@ -38,7 +38,7 @@ jobs:
     name: Run coverage tests
     needs: submodule_sanity_guard
     # Only run coverage test for draft PRs
-    if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
+    if:  false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
     uses: ./.github/workflows/coverage.yml
     with:
       duckdb_git_ref: ${{ github.base_ref }}

.github/workflows/on_push.yml

@@ -1,6 +1,6 @@
 name: Tests and coverage on push
 on:
-  push:
+  workflow_dispatch:
     branches-ignore:
       - main
       - v*.*-*

.github/workflows/packaging.yml

@@ -52,9 +52,9 @@ on:
         required: false
         type: string
 
-concurrency:
-  group: packaging-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: packaging-${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 defaults:
   run:
@@ -63,6 +63,7 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
+    if: false # disable for dev
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

.github/workflows/packaging_wheels.yml

@@ -30,14 +30,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ]
+        python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, 
         platform:
-          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          # - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
-          - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
-          - { os: macos-15,         arch: arm64,      cibw_system: macosx }
-          - { os: macos-15,         arch: universal2, cibw_system: macosx }
-          - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+          # - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          # - { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          # - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          # - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
         minimal:
           - ${{ inputs.minimal }}
         exclude:
@@ -54,27 +54,36 @@ jobs:
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
       CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+        sccache --show-stats && 
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
-
+        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs --durations=5
+      SCCACHE_GHA_ENABLED: "on"
+      SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}
+      ACTIONS_CACHE_SERVICE_V2: "1"  
+      PYTHON_GIL: "1"  
     steps:
       - name: Checkout DuckDB Python
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: ${{ inputs.duckdb-python-sha }}
           fetch-depth: 0
           submodules: true
-
       - name: Checkout DuckDB
         shell: bash
+        continue-on-error: true  # needed when we're merging into a branch
         if: ${{ inputs.duckdb-sha }}
         run: |
           cd external/duckdb
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
-
+      - name: Configure Cache Env
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
       # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
       - name: Set OVERRIDE_GIT_DESCRIBE
         shell: bash
@@ -94,9 +103,21 @@ jobs:
         env:
           CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
-          # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds. 
-          # TODO: Remove PYTHON_GIL=1 when free-threaded is supported. 
-          CIBW_ENVIRONMENT: PYTHON_GIL=1
+          # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds.
+          # TODO: Remove PYTHON_GIL=1 when free-threaded is supported.
+          CIBW_ENVIRONMENT: PYTHON_GIL=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes"
+          CIBW_ENVIRONMENT_PASS_LINUX: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL
+          CIBW_ENVIRONMENT_PASS_MACOS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL
+          CIBW_ENVIRONMENT_PASS_WINDOWS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL
+          CIBW_BEFORE_BUILD_LINUX: >
+            if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi &&
+            curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz &&
+            cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin &&
+            sccache --show-stats
+          CIBW_BEFORE_BUILD_MACOS: brew install sccache
+          CIBW_BEFORE_BUILD_WINDOWS: >
+            del "C:\Strawberry\c\bin\ccache.exe" &&
+            choco install sccache
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:

CMakeLists.txt

@@ -46,8 +46,13 @@ else()
 endif()
 duckdb_add_library(duckdb_target)
 
+# Use unity for the external duckdb_target
+set_target_properties(duckdb_target PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 32)
+
+
 # Bundle in INTERFACE library
 add_library(_duckdb_dependencies INTERFACE)
+
 target_link_libraries(_duckdb_dependencies INTERFACE
         pybind11::pybind11
         duckdb_target
@@ -77,6 +82,9 @@ pybind11_add_module(_duckdb
         $<TARGET_OBJECTS:python_relation>
         $<TARGET_OBJECTS:python_type>
 )
+# don't use unity for the duckdb-python code - we just want one file per target
+set_target_properties(_duckdb PROPERTIES UNITY_BUILD OFF) 
+
 # add _duckdb_dependencies
 target_link_libraries(_duckdb PRIVATE _duckdb_dependencies)
 

pyproject.toml

@@ -121,7 +121,7 @@ cmake.build-type = "Debug"
 if.state = "editable"
 if.env.COVERAGE = false
 inherit.cmake.define = "append"
-cmake.define.DISABLE_UNITY = "1"
+# cmake.define.DISABLE_UNITY = "1"
 
 [tool.scikit-build.sdist]
 include = [
@@ -227,6 +227,7 @@ test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    "pytest-xdist",
     "mypy",
     "coverage",
     "gcovr; python_version < '3.14'",
@@ -380,12 +381,3 @@ manylinux-pypy_x86_64-image = "manylinux_2_28"
 manylinux-aarch64-image = "manylinux_2_28"
 manylinux-pypy_aarch64-image = "manylinux_2_28"
 enable = ["cpython-freethreading", "cpython-prerelease"]
-
-[tool.cibuildwheel.linux]
-before-build = ["yum install -y ccache"]
-
-[tool.cibuildwheel.macos]
-before-build = ["brew install ccache"]
-
-[tool.cibuildwheel.windows]
-before-build = ["choco install ccache"]