Faster builds 920

.github/workflows/on_pr.yml

@@ -2,8 +2,7 @@ name: Tests and builds on PR
 on:
   pull_request:
     branches:
-      - main
-      - v*.*-*
+      - '**'
     types: [opened, reopened, ready_for_review, converted_to_draft, synchronize]
     paths-ignore:
       - '**.md'
@@ -13,19 +12,20 @@ on:
       - '.github//**'
       - '!.github/workflows/on_push.yml'
       - '!.github/workflows/coverage.yml'
+    workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
-  submodule_sanity_guard:
-    name: Make sure submodule is in a sane state
-    uses: ./.github/workflows/submodule_sanity.yml
+  # submodule_sanity_guard:
+  #   name: Make sure submodule is in a sane state
+  #   uses: ./.github/workflows/submodule_sanity.yml
 
   packaging_test:
     name: Build a minimal set of packages and run all tests on them
-    needs: submodule_sanity_guard
+    # needs: submodule_sanity_guard
     # Skip packaging tests for draft PRs
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
     uses: ./.github/workflows/packaging.yml
@@ -36,9 +36,9 @@ jobs:
 
   coverage_test:
     name: Run coverage tests
-    needs: submodule_sanity_guard
+    # needs: submodule_sanity_guard
     # Only run coverage test for draft PRs
-    if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
+    if:  false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
     uses: ./.github/workflows/coverage.yml
     with:
       duckdb_git_ref: ${{ github.base_ref }}

.github/workflows/on_push.yml

@@ -1,6 +1,6 @@
 name: Tests and coverage on push
 on:
-  push:
+  workflow_dispatch:
     branches-ignore:
       - main
       - v*.*-*

.github/workflows/packaging.yml

@@ -52,9 +52,9 @@ on:
         required: false
         type: string
 
-concurrency:
-  group: packaging-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: packaging-${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 defaults:
   run:
@@ -63,6 +63,7 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
+    if: false # disable for dev
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

.github/workflows/packaging_wheels.yml

@@ -26,55 +26,80 @@ on:
 
 jobs:
   build_wheels:
-    name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.os }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ]
+        python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314t, 
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
-          - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
-          - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
-          - { os: macos-15,         arch: arm64,      cibw_system: macosx }
-          - { os: macos-15,         arch: universal2, cibw_system: macosx }
-          - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+          - { os: ubuntu-latest,     arch: x86_64,     cibw_system: manylinux }
+          # - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          #- { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          - { os: macos-26,         arch: arm64,      cibw_system: macosx }
+          # - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          # - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
         minimal:
           - ${{ inputs.minimal }}
         exclude:
           - { minimal: true, python: cp310 }
           - { minimal: true, python: cp311 }
           - { minimal: true, python: cp312 }
           - { minimal: true, platform: { arch: universal2 } }
-          # Windows+cp314t disabled due to test failures in CI. 
-          # TODO: Diagnose why tests fail (access violations) in some configurations
-          - { python: cp314t, platform: { os: windows-2025 } }  
-
+
     runs-on: ${{ matrix.platform.os }}
     env:
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
-      CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+      CIBW_BEFORE_TEST_LINUX: >
+        unset UV_NO_BUILD_ISOLATION &&
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+        uv pip install -r pylock.toml
+      CIBW_BEFORE_TEST_MACOS: >
+        unset UV_NO_BUILD_ISOLATION &&
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
+        uv pip install -r pylock.toml
+      CIBW_BEFORE_TEST_WINDOWS: >
+        set UV_NO_BUILD_ISOLATION= &&
+        sccache --show-stats &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
+        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs --durations=5 -n 2 &&
+        uv run -v pytest ./tests/fast/threading --durations=5 --parallel-threads=10 --iterations=5 -n 2
+
+      # sccache configuration with path normalization fixes
+      SCCACHE_GHA_ENABLED: "on"
+      SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}
+      SCCACHE_BASEDIR: "/project"
+      ACTIONS_CACHE_SERVICE_V2: "1"
 
+      # Fix random temp directories for consistent caching
+      TMPDIR: "/tmp/duckdb-build"
+      TEMP: "/tmp/duckdb-build"  
     steps:
       - name: Checkout DuckDB Python
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: ${{ inputs.duckdb-python-sha }}
           fetch-depth: 0
           submodules: true
-
       - name: Checkout DuckDB
         shell: bash
+        continue-on-error: true  # needed when we're merging into a branch
         if: ${{ inputs.duckdb-sha }}
         run: |
           cd external/duckdb
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
-
+      - name: Configure Cache Env
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
       # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
       - name: Set OVERRIDE_GIT_DESCRIBE
         shell: bash
@@ -89,17 +114,109 @@ jobs:
           cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
           python-version: ${{ matrix.python }}
 
+      # # Install Astral UV, which will be used as build-frontend for cibuildwheel
+      # - name: Install UV
+      #   shell: bash
+      #   env:
+      #     TMPDIR: /tmp
+      #   run: |
+      #     curl -LsSf https://astral.sh/uv/0.8.16/install.sh | sh
+      #     echo "$HOME/.local/bin" >> $GITHUB_PATH
+      #     export PATH="$HOME/.local/bin:$PATH"
+      #     uv --version
+
+      # Load MSVC environment, needed for Windows - ninja builds
+      - uses: ilammy/msvc-dev-cmd@v1
+        if: ${{ matrix.python == 'cp314t' && matrix.platform.cibw_system == 'win' }}
+        with:
+          arch: ${{ matrix.platform.arch }}
+
       - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
         uses: pypa/[email protected]
         env:
           CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
-          # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds. 
-          # TODO: Remove PYTHON_GIL=1 when free-threaded is supported. 
-          CIBW_ENVIRONMENT: PYTHON_GIL=1
+          CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation"
+          UV_PYTHON: ${{ matrix.python }}
+          UV_PROJECT_ENVIRONMENT: /project/.venv
+          PYTHONPATH: /project
+          # SCCACHE Notes:
+          # - Without the /project dir and tmpdirs (not sure exactly which mattered), the python environment is installed to a random tmp dir, which breaks
+          # the cache key... so, only the external/duckdb would cache, and not the project itself
+          # - GHA cache is limited to 10GB LRU.
+          # - SCCACHE_BASEDIR is not implemented in sccache (https://github.com/mozilla/sccache/issues/35)
+          # - Using -fdebug-prefix-map/-fmacro-prefix-map for path normalization instead (Mozilla Firefox approach)
+          # - Mozilla bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1524662 "Support gcc/clang-like build path prefix map"
+          # - UV creates random build-env-* dirs causing cache misses (https://github.com/astral-sh/uv/issues/13096)
+          # - Using --no-isolation to eliminate random UV build environments
+          #
+          # SCCACHE_LOG: trace
+          # SCCACHE_LOG_LEVEL: trace
+          # RUST_LOG: trace
+          # SCCACHE_NO_DAEMON: "1"
+
+          # no-build-isolation uses the same build-env path for each build, stable paths needed for caching
+          CIBW_ENVIRONMENT: >
+            CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER=""
+            CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes"
+            SCCACHE_BASEDIR="/tmp/duckdb-build"
+            TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build"
+            UV_NO_BUILD_ISOLATION="true"
+            PYTHONPATH="/project"
+            UV_CACHE_DIR="/tmp/duckdb-build/uv-cache"
+            UV_PROJECT_ENVIRONMENT="/project/.venv"
+            UV_PYTHON=cp314t
+          CIBW_ENVIRONMENT_MACOS: >
+            CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER=""
+            CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes"
+            SCCACHE_BASEDIR="/tmp/duckdb-build"
+            TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build"
+            UV_NO_BUILD_ISOLATION="true"
+            PYTHONPATH="."
+            UV_CACHE_DIR="/tmp/duckdb-build/uv-cache"
+            UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/.venv"
+            UV_PYTHON=cp314t
+          CIBW_ENVIRONMENT_WINDOWS: >
+            CMAKE_BUILD_TYPE=Release CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER=""
+            CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes"
+            SCCACHE_BASEDIR="/tmp/duckdb-build"
+            TMPDIR="C:/tmp/duckdb-build" TEMP="C:/tmp/duckdb-build"
+            UV_NO_BUILD_ISOLATION="true"
+            PYTHONPATH="."
+            UV_CACHE_DIR="C:/tmp/duckdb-build/uv-cache"
+            UV_PROJECT_ENVIRONMENT="C:/project/.venv"
+            UV_PYTHON=cp314t
+
+          CIBW_ENVIRONMENT_PASS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER SCCACHE_LOG SCCACHE_LOG_LEVEL SCCACHE_NO_DAEMON RUST_LOG SCCACHE_BASEDIR TMPDIR TEMP PIP_CACHE_DIR UV_PYTHON UV_PROJECT_ENVIRONMENT PYTHONPATH UV_NO_BUILD_ISOLATION UV_CACHE_DIR
+
+          # Use pyproject.toml settings - remove overrides to let pyproject.toml before-build work
+          # Just install sccache since that's platform-specific and not in pyproject.toml
+          CIBW_BEFORE_BUILD_LINUX: >
+            mkdir -p /tmp/duckdb-build /tmp/pip-cache &&
+            if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi &&
+            curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz &&
+            cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin &&
+            sccache --show-stats &&
+            echo "Installing build dependencies..." &&
+            uv pip install --system scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10' &&
+            echo "Current directory: $(pwd)" &&
+            echo "Directory contents:" && ls -la &&
+            echo "Testing duckdb_packaging import:" &&
+            python -c "import sys; print('Python path:', sys.path); import duckdb_packaging.build_backend; print('Import successful')"
+          CIBW_BEFORE_BUILD_MACOS: >
+            mkdir -p /tmp/duckdb-build /tmp/duckdb-build/uv-cache /tmp/pip-cache &&
+            echo "Installing build dependencies..." &&
+            uv pip install scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10' &&
+            brew install sccache
+          CIBW_BEFORE_BUILD_WINDOWS: >
+            (mkdir "C:\tmp\duckdb-build" 2>nul & mkdir "C:\tmp\duckdb-build\uv-cache" 2>nul & mkdir "C:\tmp\pip-cache" 2>nul) &&
+            del "C:\Strawberry\c\bin\ccache.exe" &&
+            echo "Installing build dependencies..." &&
+            uv pip install scikit-build-core "pybind11[global]>=2.6.0" setuptools-scm "cmake>=3.29.0" "ninja>=1.10" &&
+            choco install sccache
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:
-          name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+          name: wheel-${{ matrix.python }}-${{ matrix.platform.os }}_${{ matrix.platform.arch }}
           path: wheelhouse/*.whl
           compression-level: 0