CI for 3.14 / 3.14t

.github/workflows/cleanup_pypi.yml

@@ -52,7 +52,7 @@ jobs:
       - name: Install Astral UV
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.7.14"
+          version: "0.8.16"
 
       - name: Run Cleanup
         env:

.github/workflows/coverage.yml

@@ -70,7 +70,7 @@ jobs:
       - name: Install Astral UV and enable the cache
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.7.14"
+          version: "0.8.16"
           python-version: 3.9
           enable-cache: true
           cache-suffix: -${{ github.workflow }}

.github/workflows/on_pr.yml

@@ -14,9 +14,9 @@ on:
       - '!.github/workflows/on_push.yml'
       - '!.github/workflows/coverage.yml'
 
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: ${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 jobs:
   submodule_sanity_guard:

.github/workflows/packaging.yml

@@ -52,9 +52,9 @@ on:
         required: false
         type: string
 
-concurrency:
-  group: packaging-${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: true
+# concurrency:
+#   group: packaging-${{ github.workflow }}-${{ github.ref }}
+#   cancel-in-progress: true
 
 defaults:
   run:
@@ -63,6 +63,7 @@ defaults:
 jobs:
   build_sdist:
     name: Build an sdist and determine versions
+    if: ${{ github.event_name != 'pull_request' }}
     uses: ./.github/workflows/packaging_sdist.yml
     with:
       testsuite: all

.github/workflows/packaging_sdist.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Install Astral UV
         uses: astral-sh/setup-uv@v6
         with:
-          version: "0.7.14"
+          version: "0.8.16"
           python-version: 3.11
 
       - name: Build sdist

.github/workflows/packaging_wheels.yml

@@ -27,10 +27,12 @@ on:
 jobs:
   build_wheels:
     name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}'
+    permissions:
+      actions: write  # For GitHub Actions cache write access
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313 ]
+        python: [ cp313, cp314t ]  # cp39, cp310, cp311, cp312, cp313, cp314, cp314t 
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
@@ -53,45 +55,89 @@ jobs:
         uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
-
+        sccache --show-stats && 
+        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs 
+      SCCACHE_GHA_ENABLED: "on"
+      SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }}
+      ACTIONS_CACHE_SERVICE_V2: "1"  
+      PYTHON_GIL: "1"  
+      # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions.
     steps:
       - name: Checkout DuckDB Python
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
         with:
           ref: ${{ inputs.duckdb-python-sha }}
           fetch-depth: 0
           submodules: true
-
+      # The host doesn't need sccache, only the cibuildwheel container
+      # - name: Setup sccache  # idea from https://github.com/pypa/cibuildwheel/issues/1030
+      #   uses: mozilla-actions/[email protected]
+      #   with:
+      #     version: "v0.10.0"
+      - name: Configure Cache Env
+        uses: actions/github-script@v7
+        with:
+          script: |
+            core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
       - name: Checkout DuckDB
         shell: bash
         if: ${{ inputs.duckdb-sha }}
         run: |
           cd external/duckdb
           git fetch origin
           git checkout ${{ inputs.duckdb-sha }}
-
       # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds
       - name: Set OVERRIDE_GIT_DESCRIBE
         if: ${{ inputs.set-version != '' }}
-        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
-
+        run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
       - uses: astral-sh/setup-uv@v6
         with:
-          version: "0.7.14"
-          enable-cache: false
+          version: "0.8.16"
+          enable-cache: true
+          activate-environment: true
           cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
-
-      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
-        uses: pypa/[email protected]
+          python-version: ${{
+            fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python]
+            }}
+      - name: Install cibuildwheel
+        run: |
+          echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}"
+          uv pip install cibuildwheel==3.1.4
+      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows)
+        if: matrix.platform.cibw_system == 'win'
+        shell: cmd
+        run: |
+          for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do (
+            echo Setting up x64 build environment...
+            call "%%i\VC\Auxiliary\Build\vcvars64.bat"
+            echo Verifying compiler paths:
+            where cl
+            echo VCINSTALLDIR=%VCINSTALLDIR%
+            echo Platform=%Platform%
+            uv run --no-sync python -m cibuildwheel --output-dir wheelhouse
+          )
         env:
           CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
-
+          CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON
+          CIBW_BUILD_VERBOSITY: 3
+      - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows)
+        if: matrix.platform.cibw_system != 'win'
+        run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse
+        env:
+          CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
+          CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
+          CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" 
+          CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER=""
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: |
+            sccache --show-stats
+            auditwheel repair -w {dest_dir} {wheel}
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:
           name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
           path: wheelhouse/*.whl
-          compression-level: 0
+          compression-level: 0

pyproject.toml

@@ -46,9 +46,10 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l
     "ipython", # used in duckdb.query_graph
     "fsspec",  # used in duckdb.filesystem
     "numpy",   # used in duckdb.experimental.spark and in duckdb.fetchnumpy()
-    "pandas",  # used for pandas dataframes all over the place
-    "pyarrow", # used for pyarrow support
-    "adbc_driver_manager", # for the adbc driver (TODO: this should live under the duckdb package)
+    "pandas; python_version < '3.14'",  # used for pandas dataframes all over the place
+    "pandas>=2.3.0.dev0; python_version >= '3.14'",  # nightly build for Python 3.14
+    "pyarrow; python_version < '3.14'", # used for pyarrow support
+    "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package)
 ]
 
 ######################################################################################################
@@ -204,6 +205,7 @@ required-environments = [ # ... but do always resolve for all of them
     "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'x86_64'",
     "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'aarch64'",
 ]
+prerelease = "allow" # for 3.14
 
 # We just need pytorch for tests, wihtout GPU acceleration. PyPI doesn't host a cpu-only version for Linux, so we have
 # to configure the index url for cpu-only pytorch manually
@@ -212,6 +214,11 @@ name = "pytorch-cpu"
 url = "https://download.pytorch.org/whl/cpu"
 explicit = true
 
+[[tool.uv.index]]
+name = "scientific-python-nightly"
+url = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple"
+explicit = true
+
 [tool.uv.sources]
 torch = [ { index = "pytorch-cpu" } ]
 torchvision = [ { index = "pytorch-cpu" } ]
@@ -227,23 +234,27 @@ test = [ # dependencies used for running tests
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    # "pytest-randomly",      # Ensures non-deterministic test order
+    # "pytest-xdist",      # Parallel tests (separate processes)
+    # "pytest-run-parallel",  # For free-threading testing
     "mypy",
     "coverage",
-    "gcovr",
+    "gcovr; python_version < '3.14'",
     "gcsfs",
     "packaging",
-    "polars",
+    "polars; python_version < '3.14'",
     "psutil",
     "py4j",
     "pyotp",
-    "pyspark",
+    "pyspark; python_version < '3.14'",
     "pytz",
     "requests",
     "urllib3",
     "fsspec>=2022.11.0",
-    "pandas>=2.0.0",
-    "pyarrow>=18.0.0",
-    "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'",
+    "pandas>=2.0.0; python_version < '3.14'",
+    "pandas>=2.3.0.dev0; python_version >= '3.14'",
+    "pyarrow>=18.0.0; python_version < '3.14'",
+    "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')",
     "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'",
     "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'",
     "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'",
@@ -256,10 +267,11 @@ scripts = [ # dependencies used for running scripts
     "ipython",
     "ipywidgets",
     "numpy",
-    "pandas",
+    "pandas; python_version < '3.14'",
+    "pandas>=2.3.0.dev0; python_version >= '3.14'",
     "pcpp",
-    "polars",
-    "pyarrow",
+    "polars; python_version < '3.14'",
+    "pyarrow; python_version < '3.14'",
     "pytz"
 ]
 pypi = [ # dependencies used by the pypi cleanup script
@@ -305,6 +317,7 @@ filterwarnings = [
     # Pyspark is throwing these warnings
     "ignore:distutils Version classes are deprecated:DeprecationWarning",
     "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning",
+    "ignore:ChainedAssignmentError.*:FutureWarning"
 ]
 
 [tool.coverage.run]
@@ -379,12 +392,111 @@ manylinux-x86_64-image = "manylinux_2_28"
 manylinux-pypy_x86_64-image = "manylinux_2_28"
 manylinux-aarch64-image = "manylinux_2_28"
 manylinux-pypy_aarch64-image = "manylinux_2_28"
+enable = ["cpython-freethreading", "cpython-prerelease"]
 
 [tool.cibuildwheel.linux]
-before-build = ["yum install -y ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+before-build = [
+  "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi",
+  "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz",
+  "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin",
+  "which sccache",
+  "echo 'Compiler launcher detection order check:'",
+  "which ccache || echo 'ccache not found'",
+  "which sccache || echo 'sccache not found'", 
+  "echo 'PATH priority test:'",
+  "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done",
+  "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config",
+  "mkdir -p /tmp/sccache-cache",
+  "export SCCACHE_DIR=/tmp/sccache-cache",
+  "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi",
+  "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi",
+  "echo \"GitHub Actions environment variables verified in container\"",
+  "echo \"Testing sccache GitHub Actions cache access...\"", 
+  "sccache --start-server",
+  "SCCACHE_LOG=debug sccache --show-stats 2>&1 | head -10 || true",
+  "echo 'int main(){return 0;}' > test.c",
+  "SCCACHE_LOG=debug sccache gcc -c test.c -o test.o 2>&1 | head -20 || true",
+  "sccache --show-stats",
+  "echo \"Sccache test complete\""
+]
+before-test = "sccache --show-stats"
 
 [tool.cibuildwheel.macos]
-before-build = ["brew install ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+before-build = [
+    "brew install sccache",
+    "echo 'Compiler launcher detection order check:'",
+    "which ccache || echo 'ccache not found'",
+    "which sccache || echo 'sccache not found'",
+    "echo 'PATH priority test:'",
+    "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done"
+]
 
 [tool.cibuildwheel.windows]
-before-build = ["choco install ccache"]
+environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"]
+# if ccache is in the path, it's chosen before sccache
+# and, uninstalling strawberry is much slower than just deleting the entry
+before-build = [
+    "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"",
+    "choco install sccache",
+    "sccache --version"
+]
+before-test = [
+    "sccache --show-stats"
+]
+
+[[tool.scikit-build.overrides]]
+# Regular Windows builds (non-free-threading)
+if.platform-system = "^win32"
+if.abi-flags = ""
+cmake.args = [
+    "-G", "Ninja",
+    "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
+    "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded",
+    "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG",
+    "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG",
+    "-DCMAKE_SYSTEM_PROCESSOR=AMD64",
+    "-DCMAKE_C_COMPILER_LAUNCHER=",
+    "-DCMAKE_CXX_COMPILER_LAUNCHER=",
+    "-DCORE_EXTENSIONS=",
+    "-DBUILD_UNITTESTS=OFF"
+]
+
+[[tool.scikit-build.overrides]]
+# Free-threading Windows builds (Python 3.14t)
+# To get sccache to work, needed a few things
+# Ninja
+# /Z7 so we don't write parallel PDBs
+# The empty COMPILER_LAUNCHERS are because otherwise sccache is called twice (sccache sccache cl.exe ...)
+# The LIBRARY is because it picks up python314.lib, instead of python314t.lib
+# TODO: Dynamically add the "t" for future versions (ie: python315t)
+if.platform-system = "^win32"
+if.abi-flags = "t"
+cmake.args = [
+    "-G", "Ninja",
+    "-DCMAKE_BUILD_TYPE=RelWithDebInfo",
+    "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded",
+    "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED",
+    "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED",
+    "-DCMAKE_C_COMPILER_LAUNCHER=",
+    "-DCMAKE_CXX_COMPILER_LAUNCHER=",
+]
+    #    "-DCMAKE_SYSTEM_PROCESSOR=AMD64",
+
+    #"-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib",
+    #"-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib",
+
+    #    "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}",
+
+    #"-DPython3_FIND_STRATEGY=LOCATION",
+    #"-DPython3_FIND_ABI=ANY;ANY;ANY;ON",
+    #"-DPython3_LIBRARY_RELEASE=python314t.lib",
+    # "-DCORE_EXTENSIONS=",
+    # "-DBUILD_UNITTESTS=OFF"
+    # "--debug-output",
+    # "--debug-find",
+    # "--debug-find-pkg=Python3",
+    # "--trace-expand",
+    # "--log-level=DEBUG"
+

test_mre.py

@@ -0,0 +1,11 @@
+import pandas as pd
+
+
+def test_chained_assignment_warning():
+    """This should trigger the ChainedAssignmentError warning we're trying to suppress"""
+    df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
+
+    # This chained assignment triggers the warning
+    df['A'][0] = 999
+
+    assert df['A'][0] == 999