Add Python 3.14 support (duckdb#116)

Adding 3.14 support:
- Catching import errors for pyarrow.
- Moving from the deprecated `typing._UnionGenericAlias` to check
whether a given object is a Union to `isinstance(obj, types.UnionType)`
for >= py310 and `typing.get_origin(obj) is typing.Union` as fallback.
This meant running the importcache code generation scripts, which needed
to output a new header (and had a small bug).
- Moving from the
[deprecated](https://pandas.pydata.org/docs/user_guide/indexing.html#why-does-assignment-fail-when-using-chained-indexing)
[pandas chained
assignment](https://pandas.pydata.org/docs/user_guide/copy_on_write.html#copy-on-write-chained-assignment)
- which will never work with Copy-on-Write enabled in Pandas, which in
turn will be the default from pandas 3.0 onwards - to a delete-and-write
approach for `tz_localize` columns in Pandas. Apart from the chained
assignment issue, the tz is now also explicitly part of the column's
datatype, and pandas doesn't like it when we change the timezone and
throws a `FutureWarning: Setting an item of incompatible dtype is
deprecated and will raise in a future error of pandas. Value
'<DatetimeArray>`.
- Remove py313 from the "fast" sanity check of the packaging workflow.
- Added `--quiet` to `uv export` in packaging_wheels. The output has
been very useful to set the env markers correctly for all dependencies
(which is much better now than it was in-tree) but I agree it's a bit
noisy. It was already set for sdists, so may as well have it here as
well.

Author: evertlammerts

.github/workflows/cleanup_pypi.yml

@@ -50,9 +50,9 @@ jobs:
           exit 1
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
 
       - name: Run Cleanup
         env:

.github/workflows/code_quality.yml

@@ -29,9 +29,9 @@ jobs:
           persist-credentials: false
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
           python-version: 3.9
 
       - name: pre-commit (cache)

.github/workflows/coverage.yml

@@ -68,9 +68,9 @@ jobs:
           sudo apt-get -y install ccache
 
       - name: Install Astral UV and enable the cache
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
           python-version: 3.9
           enable-cache: true
           cache-suffix: -${{ github.workflow }}
@@ -79,7 +79,7 @@ jobs:
         shell: bash
         run: |
           if [[ "${{ inputs.testsuite }}" == "all" ]]; then
-            uv run coverage run -m pytest ./tests --ignore=./tests/stubs
+            uv run coverage run -m pytest ./tests
           elif [[ "${{ inputs.testsuite }}" == "fast" ]]; then
             uv run coverage run -m pytest ./tests/fast
           else

.github/workflows/packaging_sdist.yml

@@ -56,9 +56,9 @@ jobs:
         run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
 
       - name: Install Astral UV
-        uses: astral-sh/setup-uv@v6
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
           python-version: 3.11
 
       - name: Build sdist
@@ -80,7 +80,7 @@ jobs:
           # run tests
           tests_root="${{ github.workspace }}/tests"
           tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}"
-          uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs
+          uv run --verbose pytest -c ${{ github.workspace }}/pyproject.toml $tests_dir
 
       - id: versioning
         run: |

.github/workflows/packaging_wheels.yml

@@ -30,7 +30,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [ cp39, cp310, cp311, cp312, cp313 ]
+        python: [ cp39, cp310, cp311, cp312, cp313, cp314 ]
         platform:
           - { os: windows-2025,     arch: amd64,      cibw_system: win }
           - { os: ubuntu-24.04,     arch: x86_64,     cibw_system: manylinux }
@@ -44,16 +44,29 @@ jobs:
           - { minimal: true, python: cp310 }
           - { minimal: true, python: cp311 }
           - { minimal: true, python: cp312 }
+          - { minimal: true, python: cp313 }
           - { minimal: true, platform: { arch: universal2 } }
     runs-on: ${{ matrix.platform.os }}
     env:
+      ### cibuildwheel configuration
+      #
+      # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others):
+      # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it
+      #   built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make
+      #   sure that the pytest config in pyproject.toml is available.
+      # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing,
+      #   `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments
+      #   across all platforms we build for. This might be solved in newer uv versions.
+      # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from
+      #   traversing the full filesystem, which produces an error on Windows.
+      # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64.
       CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }}
       CIBW_TEST_SOURCES: tests
       CIBW_BEFORE_TEST: >
-        uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} &&
+        uv export --only-group test --no-emit-project  --quiet --output-file pylock.toml --directory {project} &&
         uv pip install -r pylock.toml
       CIBW_TEST_COMMAND: >
-        uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs
+        uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }}
 
     steps:
       - name: Checkout DuckDB Python
@@ -78,14 +91,14 @@ jobs:
         run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV
 
       # Install Astral UV, which will be used as build-frontend for cibuildwheel
-      - uses: astral-sh/setup-uv@v6
+      - uses: astral-sh/setup-uv@v7
         with:
-          version: "0.7.14"
+          version: "0.9.0"
           enable-cache: false
           cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}
 
       - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels
-        uses: pypa/cibuildwheel@v3.0
+        uses: pypa/cibuildwheel@v3.2
         env:
           CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }}
           CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}

cmake/compiler_launcher.cmake

@@ -8,8 +8,7 @@ include(CMakeParseArguments)
 # Function to look for ccache and sccache to speed up builds, if available
 # ────────────────────────────────────────────
 function(setup_compiler_launcher_if_available)
-  if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED
-                                               ENV{CMAKE_C_COMPILER_LAUNCHER})
+  if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER)
     find_program(COMPILER_LAUNCHER NAMES ccache sccache)
     if(COMPILER_LAUNCHER)
       message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher")
@@ -19,8 +18,7 @@ function(setup_compiler_launcher_if_available)
     endif()
   endif()
 
-  if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER
-     AND NOT DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER})
+  if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER)
     find_program(COMPILER_LAUNCHER NAMES ccache sccache)
     if(COMPILER_LAUNCHER)
       message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher")

pyproject.toml

@@ -47,7 +47,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l
     "fsspec",  # used in duckdb.filesystem
     "numpy",   # used in duckdb.experimental.spark and in duckdb.fetchnumpy()
     "pandas",  # used for pandas dataframes all over the place
-    "pyarrow", # used for pyarrow support
+    "pyarrow; python_version < '3.14'", # used for pyarrow support
     "adbc-driver-manager", # for the adbc driver
 ]
 
@@ -226,13 +226,14 @@ stubdeps = [ # dependencies used for typehints in the stubs
     "fsspec",
     "pandas",
     "polars",
-    "pyarrow",
+    "pyarrow; python_version < '3.14'",
 ]
 test = [ # dependencies used for running tests
     "adbc-driver-manager",
     "pytest",
     "pytest-reraise",
     "pytest-timeout",
+    "pytest-timestamper",
     "mypy",
     "coverage",
     "gcovr",
@@ -248,8 +249,8 @@ test = [ # dependencies used for running tests
     "urllib3",
     "fsspec>=2022.11.0",
     "pandas>=2.0.0",
-    "pyarrow>=18.0.0",
-    "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'",
+    "pyarrow>=18.0.0; python_version < '3.14'",
+    "torch>=2.2.2; python_version < '3.14' and ( sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13' )",
     "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'",
     "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'",
     "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'",
@@ -265,7 +266,7 @@ scripts = [ # dependencies used for running scripts
     "pandas",
     "pcpp",
     "polars",
-    "pyarrow",
+    "pyarrow; python_version < '3.14'",
     "pytz"
 ]
 pypi = [ # dependencies used by the pypi cleanup script
@@ -302,7 +303,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "-ra -q"
+addopts = "-ra --verbose"
 testpaths = ["tests"]
 filterwarnings = [
     "error",

scripts/cache_data.json

@@ -622,7 +622,8 @@
         "full_path": "typing",
         "name": "typing",
         "children": [
-            "typing._UnionGenericAlias"
+            "typing.Union",
+            "typing.get_origin"
         ]
     },
     "typing._UnionGenericAlias": {
@@ -793,5 +794,17 @@
         "full_path": "pyarrow.decimal128",
         "name": "decimal128",
         "children": []
+    },
+    "typing.get_origin": {
+        "type": "attribute",
+        "full_path": "typing.get_origin",
+        "name": "get_origin",
+        "children": []
+    },
+    "typing.Union": {
+        "type": "attribute",
+        "full_path": "typing.Union",
+        "name": "Union",
+        "children": []
     }
 }

scripts/generate_import_cache_cpp.py

@@ -151,10 +151,10 @@ def to_string(self):
 
 //! Note: This class is generated using scripts.
 //! If you need to add a new object to the cache you must:
-//! 1. adjust tools/pythonpkg/scripts/imports.py
-//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py
-//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py
-//! 4. run make format-main (the generator doesn't respect the formatting rules ;))
+//! 1. adjust scripts/imports.py
+//! 2. run python scripts/generate_import_cache_json.py
+//! 3. run python scripts/generate_import_cache_cpp.py
+//! 4. run pre-commit to fix formatting errors
 
 namespace duckdb {{
 {self.get_classes()}
@@ -230,7 +230,7 @@ def get_root_modules(files: list[ModuleFile]):
 
 
 def get_module_file_path_includes(files: list[ModuleFile]):
-    template = '#include "duckdb_python/import_cache/modules/{}'
+    template = '#include "duckdb_python/import_cache/modules/{}"'
     return "\n".join(template.format(f.file_name) for f in files)
 
 

scripts/imports.py

@@ -128,7 +128,8 @@
 
 import typing
 
-typing._UnionGenericAlias
+typing.Union
+typing.get_origin
 
 import uuid