Ci314t - Faster CI w/ sccache and xdist

.github/workflows/on_pr.yml

@@ -2,8 +2,7 @@ name: Tests and builds on PR
 on:
   pull_request:
     branches:
-      - main
-      - v*.*-*
+      - '**'
     types: [opened, reopened, ready_for_review, converted_to_draft, synchronize]
     paths-ignore:
       - '**.md'
@@ -13,10 +12,11 @@ on:
       - '.github//**'
       - '!.github/workflows/on_push.yml'
       - '!.github/workflows/coverage.yml'
+    workflow_dispatch:
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
   submodule_sanity_guard:
@@ -38,7 +38,7 @@ jobs:
     name: Run coverage tests
     needs: submodule_sanity_guard
     # Only run coverage test for draft PRs
-    if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
+    if:  false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
     uses: ./.github/workflows/coverage.yml
     with:
       duckdb_git_ref: ${{ github.base_ref }}

.github/workflows/on_push.yml

@@ -1,24 +1,12 @@
 name: Tests and coverage on push
-on:
-  push:
-    branches-ignore:
-      - main
-      - v*.*-*
-    paths-ignore:
-      - '**.md'
-      - 'LICENSE'
-      - '.editorconfig'
-      - 'scripts/**'
-      - '.github//**'
-      - '!.github/workflows/on_push.yml'
-      - '!.github/workflows/coverage.yml'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
   test:
+    if:  false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }}
     name: Run coverage tests
     uses: ./.github/workflows/coverage.yml
     with:

.github/workflows/packaging.yml

@@ -52,9 +52,9 @@ on:
         required: false
         type: string
 
-concurrency:
-  group: packaging-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: packaging-${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 defaults:
   run:
@@ -63,6 +63,7 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
+    if: false # disable for dev
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

.github/workflows/packaging_wheels.yml

@@ -30,14 +30,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ]
+        python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, 
         platform:
-          - { os: windows-2025,     arch: amd64,      cibw_system: win }
+          # - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
-          - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
-          - { os: macos-15,         arch: arm64,      cibw_system: macosx }
-          - { os: macos-15,         arch: universal2, cibw_system: macosx }
-          - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
+          # - { os: ubuntu-24.04-arm, arch: aarch64,    cibw_system: manylinux }
+          # - { os: macos-15,         arch: arm64,      cibw_system: macosx }
+          # - { os: macos-15,         arch: universal2, cibw_system: macosx }
+          # - { os: macos-13,         arch: x86_64,     cibw_system: macosx }
         minimal:
           - ${{ inputs.minimal }}
         exclude:
@@ -54,27 +54,35 @@ jobs:
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
       CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
-
+        uv run -v pytest -n auto ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
+      SCCACHE_GHA_ENABLED: "on"
+      SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}
+      ACTIONS_CACHE_SERVICE_V2: "1"  
+      PYTHON_GIL: "1"  
     steps:
       - name: Checkout DuckDB Python
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: ${{ inputs.duckdb-python-sha }}
           fetch-depth: 0
           submodules: true
-
       - name: Checkout DuckDB
         shell: bash
+        continue-on-error: true  # needed when we're merging into a branch
         if: ${{ inputs.duckdb-sha }}
         run: |
           cd external/duckdb
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
-
+      - name: Configure Cache Env
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
       # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
       - name: Set OVERRIDE_GIT_DESCRIBE
         shell: bash
@@ -96,7 +104,7 @@ jobs:
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
           # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds. 
           # TODO: Remove PYTHON_GIL=1 when free-threaded is supported. 
-          CIBW_ENVIRONMENT: PYTHON_GIL=1
+          CIBW_ENVIRONMENT: PYTHON_GIL=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" 
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:

pyproject.toml

@@ -227,6 +227,7 @@ test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    "pytest-xdist",
     "mypy",
     "coverage",
     "gcovr; python_version < '3.14'",
@@ -382,10 +383,36 @@ manylinux-pypy_aarch64-image = "manylinux_2_28"
 enable = ["cpython-freethreading", "cpython-prerelease"]
 
 [tool.cibuildwheel.linux]
-before-build = ["yum install -y ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+before-build = [
+  "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi",
+  "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz",
+  "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin",
+  "sccache --show-stats"]
+before-test = ["sccache --show-stats"]
 
 [tool.cibuildwheel.macos]
-before-build = ["brew install ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+before-build = ["brew install sccache"]
 
 [tool.cibuildwheel.windows]
-before-build = ["choco install ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+before-build = [
+     "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"",
+     "choco install sccache"]
+
+[[tool.scikit-build.overrides]]
+# Windows Free-Threading 
+if.platform-system = "^win32"
+if.abi-flags = "t"
+inherit.cmake.define = "append"
+cmake.define.CMAKE_BUILD_TYPE=""
+cmake.define.CMAKE_MSVC_DEBUG_INFORMATION_FORMAT="Embedded"
+cmake.define.CMAKE_C_FLAGS="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED"
+cmake.define.CMAKE_CXX_FLAGS="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED"
+cmake.args = [
+   "-G", "Ninja",
+   "--log-level=DEBUG",
+]
+cmake.define.CMAKE_C_COMPILER_LAUNCHER=""
+cmake.define.CMAKE_CXX_COMPILER_LAUNCHER=""