diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml index c4300be3..34fe3b53 100644 --- a/.github/workflows/cleanup_pypi.yml +++ b/.github/workflows/cleanup_pypi.yml @@ -50,9 +50,9 @@ jobs: exit 1 - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" - name: Run Cleanup env: diff --git a/.github/workflows/code_quality.yml b/.github/workflows/code_quality.yml index 719e54ca..99b7884c 100644 --- a/.github/workflows/code_quality.yml +++ b/.github/workflows/code_quality.yml @@ -29,9 +29,9 @@ jobs: persist-credentials: false - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.9 - name: pre-commit (cache) diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 41880382..fd62b6c1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -68,9 +68,9 @@ jobs: sudo apt-get -y install ccache - name: Install Astral UV and enable the cache - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.9 enable-cache: true cache-suffix: -${{ github.workflow }} @@ -79,7 +79,7 @@ jobs: shell: bash run: | if [[ "${{ inputs.testsuite }}" == "all" ]]; then - uv run coverage run -m pytest ./tests --ignore=./tests/stubs + uv run coverage run -m pytest ./tests elif [[ "${{ inputs.testsuite }}" == "fast" ]]; then uv run coverage run -m pytest ./tests/fast else diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 2723b437..b6558744 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -56,9 +56,9 @@ jobs: run: echo "OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - name: Install Astral UV - uses: astral-sh/setup-uv@v6 + uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" python-version: 3.11 - name: Build sdist @@ -80,7 +80,7 @@ jobs: # run tests tests_root="${{ github.workspace }}/tests" tests_dir="${tests_root}${{ inputs.testsuite == 'fast' && '/fast' || '/' }}" - uv run --verbose pytest $tests_dir --verbose --ignore=${tests_root}/stubs + uv run --verbose pytest -c ${{ github.workspace }}/pyproject.toml $tests_dir - id: versioning run: | diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index b1a393a1..a2e4f857 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,7 +30,7 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313 ] + python: [ cp39, cp310, cp311, cp312, cp313, cp314 ] platform: - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } @@ -44,16 +44,29 @@ jobs: - { minimal: true, python: cp310 } - { minimal: true, python: cp311 } - { minimal: true, python: cp312 } + - { minimal: true, python: cp313 } - { minimal: true, platform: { arch: universal2 } } runs-on: ${{ matrix.platform.os }} env: + ### cibuildwheel configuration + # + # This is somewhat brittle, so be careful with changes. Some notes for our future selves (and others): + # - cibw will change its cwd to a temp dir and create a separate venv for testing. It then installs the wheel it + # built into that venv, and run the CIBW_TEST_COMMAND. We have to install all dependencies ourselves, and make + # sure that the pytest config in pyproject.toml is available. + # - CIBW_BEFORE_TEST installs the test dependencies by exporting them into a pylock.toml. At the time of writing, + # `uv sync --no-install-project` had problems correctly resolving dependencies using resolution environments + # across all platforms we build for. This might be solved in newer uv versions. + # - CIBW_TEST_COMMAND specifies pytest conf from pyproject.toml. --confcutdir is needed to prevent pytest from + # traversing the full filesystem, which produces an error on Windows. + # - CIBW_TEST_SKIP we always skip tests for *-macosx_universal2 builds, because we run tests for arm64 and x86_64. CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && + uv export --only-group test --no-emit-project --quiet --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + uv run -v pytest --confcutdir=. --rootdir . -c {project}/pyproject.toml ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} steps: - name: Checkout DuckDB Python @@ -78,14 +91,14 @@ jobs: run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - - uses: astral-sh/setup-uv@v6 + - uses: astral-sh/setup-uv@v7 with: - version: "0.7.14" + version: "0.9.0" enable-cache: false cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.0 + uses: pypa/cibuildwheel@v3.2 env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} diff --git a/cmake/compiler_launcher.cmake b/cmake/compiler_launcher.cmake index 8f77da86..faa4b45c 100644 --- a/cmake/compiler_launcher.cmake +++ b/cmake/compiler_launcher.cmake @@ -8,8 +8,7 @@ include(CMakeParseArguments) # Function to look for ccache and sccache to speed up builds, if available # ──────────────────────────────────────────── function(setup_compiler_launcher_if_available) - if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER AND NOT DEFINED - ENV{CMAKE_C_COMPILER_LAUNCHER}) + if(NOT DEFINED CMAKE_C_COMPILER_LAUNCHER) find_program(COMPILER_LAUNCHER NAMES ccache sccache) if(COMPILER_LAUNCHER) message(STATUS "Using ${COMPILER_LAUNCHER} as C compiler launcher") @@ -19,8 +18,7 @@ function(setup_compiler_launcher_if_available) endif() endif() - if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER - AND NOT DEFINED ENV{CMAKE_CXX_COMPILER_LAUNCHER}) + if(NOT DEFINED CMAKE_CXX_COMPILER_LAUNCHER) find_program(COMPILER_LAUNCHER NAMES ccache sccache) if(COMPILER_LAUNCHER) message(STATUS "Using ${COMPILER_LAUNCHER} as C++ compiler launcher") diff --git a/pyproject.toml b/pyproject.toml index 804c1ade..af75d9a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas", # used for pandas dataframes all over the place - "pyarrow", # used for pyarrow support + "pyarrow; python_version < '3.14'", # used for pyarrow support "adbc-driver-manager", # for the adbc driver ] @@ -226,13 +226,14 @@ stubdeps = [ # dependencies used for typehints in the stubs "fsspec", "pandas", "polars", - "pyarrow", + "pyarrow; python_version < '3.14'", ] test = [ # dependencies used for running tests "adbc-driver-manager", "pytest", "pytest-reraise", "pytest-timeout", + "pytest-timestamper", "mypy", "coverage", "gcovr", @@ -248,8 +249,8 @@ test = [ # dependencies used for running tests "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0", - "pyarrow>=18.0.0", - "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", + "pyarrow>=18.0.0; python_version < '3.14'", + "torch>=2.2.2; python_version < '3.14' and ( sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13' )", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'", @@ -265,7 +266,7 @@ scripts = [ # dependencies used for running scripts "pandas", "pcpp", "polars", - "pyarrow", + "pyarrow; python_version < '3.14'", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script @@ -302,7 +303,7 @@ dev = [ # tooling like uv will install this automatically when syncing the envir [tool.pytest.ini_options] minversion = "6.0" -addopts = "-ra -q" +addopts = "-ra --verbose" testpaths = ["tests"] filterwarnings = [ "error", diff --git a/scripts/cache_data.json b/scripts/cache_data.json index 3dd9a1f1..b1f1df3e 100644 --- a/scripts/cache_data.json +++ b/scripts/cache_data.json @@ -622,7 +622,8 @@ "full_path": "typing", "name": "typing", "children": [ - "typing._UnionGenericAlias" + "typing.Union", + "typing.get_origin" ] }, "typing._UnionGenericAlias": { @@ -793,5 +794,17 @@ "full_path": "pyarrow.decimal128", "name": "decimal128", "children": [] + }, + "typing.get_origin": { + "type": "attribute", + "full_path": "typing.get_origin", + "name": "get_origin", + "children": [] + }, + "typing.Union": { + "type": "attribute", + "full_path": "typing.Union", + "name": "Union", + "children": [] } } \ No newline at end of file diff --git a/scripts/generate_import_cache_cpp.py b/scripts/generate_import_cache_cpp.py index d10dde0c..04efffde 100644 --- a/scripts/generate_import_cache_cpp.py +++ b/scripts/generate_import_cache_cpp.py @@ -151,10 +151,10 @@ def to_string(self): //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb {{ {self.get_classes()} @@ -230,7 +230,7 @@ def get_root_modules(files: list[ModuleFile]): def get_module_file_path_includes(files: list[ModuleFile]): - template = '#include "duckdb_python/import_cache/modules/{}' + template = '#include "duckdb_python/import_cache/modules/{}"' return "\n".join(template.format(f.file_name) for f in files) diff --git a/scripts/imports.py b/scripts/imports.py index d7e38750..ee4c4597 100644 --- a/scripts/imports.py +++ b/scripts/imports.py @@ -128,7 +128,8 @@ import typing -typing._UnionGenericAlias +typing.Union +typing.get_origin import uuid diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp index 08bce28d..d8e3ad03 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/collections_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp index 508d5133..c05f77d4 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/datetime_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp index 15f4ecf6..7c979329 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/decimal_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp index 2272a7d7..29a6dc3f 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/duckdb_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp index 2742e19b..91a16fb6 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipython_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp index 26916b8d..0b9a10f1 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/ipywidgets_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp index 041e854b..e67ae893 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/numpy_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp index ef0de3b4..05ff25e2 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pandas_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp index 1ef2d473..1025f487 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pathlib_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp index 04db331b..c22173b4 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/polars_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp index d3331565..ff9d9ebc 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pyarrow_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp index 1aca4e2e..ccdde0c8 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/pytz_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp index f88b8a32..9de7dc3a 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/types_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp index 7ebf3696..38e29331 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/typing_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { @@ -26,12 +26,13 @@ struct TypingCacheItem : public PythonImportCacheItem { static constexpr const char *Name = "typing"; public: - TypingCacheItem() : PythonImportCacheItem("typing"), _UnionGenericAlias("_UnionGenericAlias", this) { + TypingCacheItem() : PythonImportCacheItem("typing"), Union("Union", this), get_origin("get_origin", this) { } ~TypingCacheItem() override { } - PythonImportCacheItem _UnionGenericAlias; + PythonImportCacheItem Union; + PythonImportCacheItem get_origin; }; } // namespace duckdb diff --git a/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp b/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp index 7bd3a6c2..f3108848 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/modules/uuid_module.hpp @@ -13,10 +13,10 @@ //! Note: This class is generated using scripts. //! If you need to add a new object to the cache you must: -//! 1. adjust tools/pythonpkg/scripts/imports.py -//! 2. run python3 tools/pythonpkg/scripts/generate_import_cache_json.py -//! 3. run python3 tools/pythonpkg/scripts/generate_import_cache_cpp.py -//! 4. run make format-main (the generator doesn't respect the formatting rules ;)) +//! 1. adjust scripts/imports.py +//! 2. run python scripts/generate_import_cache_json.py +//! 3. run python scripts/generate_import_cache_cpp.py +//! 4. run pre-commit to fix formatting errors namespace duckdb { diff --git a/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp b/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp index 172c0513..e82cd2dc 100644 --- a/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp +++ b/src/duckdb_py/include/duckdb_python/import_cache/python_import_cache_modules.hpp @@ -12,4 +12,4 @@ #include "duckdb_python/import_cache/modules/types_module.hpp" #include "duckdb_python/import_cache/modules/typing_module.hpp" #include "duckdb_python/import_cache/modules/uuid_module.hpp" -#include "duckdb_python/import_cache/modules/collections_module.hpp" +#include "duckdb_python/import_cache/modules/collections_module.hpp" \ No newline at end of file diff --git a/src/duckdb_py/pyresult.cpp b/src/duckdb_py/pyresult.cpp index a2607a12..43edf0e1 100644 --- a/src/duckdb_py/pyresult.cpp +++ b/src/duckdb_py/pyresult.cpp @@ -23,6 +23,8 @@ #include "duckdb/main/chunk_scan_state/query_result.hpp" #include "duckdb/common/arrow/arrow_query_result.hpp" +using namespace pybind11::literals; + namespace duckdb { DuckDBPyResult::DuckDBPyResult(unique_ptr result_p) : result(std::move(result_p)) { @@ -293,8 +295,10 @@ void DuckDBPyResult::ChangeToTZType(PandasDataFrame &df) { if (result->types[i] == LogicalType::TIMESTAMP_TZ) { // first localize to UTC then convert to timezone_config auto utc_local = df[names[i].c_str()].attr("dt").attr("tz_localize")("UTC"); - df.attr("__setitem__")(names[i].c_str(), - utc_local.attr("dt").attr("tz_convert")(result->client_properties.time_zone)); + auto new_value = utc_local.attr("dt").attr("tz_convert")(result->client_properties.time_zone); + // We need to create the column anew because the exact dt changed to a new timezone + df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); + df.attr("__setitem__")(names[i].c_str(), new_value); } } } @@ -378,7 +382,9 @@ PandasDataFrame DuckDBPyResult::FrameFromNumpy(bool date_as_object, const py::ha if (date_as_object) { for (idx_t i = 0; i < result->ColumnCount(); i++) { if (result->types[i] == LogicalType::DATE) { - df.attr("__setitem__")(names[i].c_str(), df[names[i].c_str()].attr("dt").attr("date")); + auto new_value = df[names[i].c_str()].attr("dt").attr("date"); + df.attr("drop")("columns"_a = names[i].c_str(), "inplace"_a = true); + df.attr("__setitem__")(names[i].c_str(), new_value); } } } diff --git a/src/duckdb_py/typing/pytype.cpp b/src/duckdb_py/typing/pytype.cpp index eca92bed..e7e31a18 100644 --- a/src/duckdb_py/typing/pytype.cpp +++ b/src/duckdb_py/typing/pytype.cpp @@ -20,17 +20,16 @@ bool PyGenericAlias::check_(const py::handle &object) { // NOLINTNEXTLINE(readability-identifier-naming) bool PyUnionType::check_(const py::handle &object) { auto types_loaded = ModuleIsLoaded(); - auto typing_loaded = ModuleIsLoaded(); - - if (!types_loaded && !typing_loaded) { - return false; - } - auto &import_cache = *DuckDBPyConnection::ImportCache(); + + // for >= py310: isinstance(object, types.UnionType) if (types_loaded && py::isinstance(object, import_cache.types.UnionType())) { return true; } - if (typing_loaded && py::isinstance(object, import_cache.typing._UnionGenericAlias())) { + // for all py3: typing.get_origin(object) is typing.Union + auto get_origin_func = import_cache.typing.get_origin(); + auto origin = get_origin_func(object); + if (origin.is(import_cache.typing.Union())) { return true; } return false; diff --git a/tests/conftest.py b/tests/conftest.py index bfb458a5..f2ad7cec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +import sys import warnings from importlib import import_module from pathlib import Path @@ -35,6 +36,47 @@ def import_pandas(): pytest.skip("Couldn't import pandas") +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item): + """Convert missing pyarrow imports to skips. + + TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. + https://github.com/duckdblabs/duckdb-internal/issues/6182 + """ + outcome = yield + if sys.version_info[:2] == (3, 14): + try: + outcome.get_result() + except ImportError as e: + if e.name == "pyarrow": + pytest.skip(f"pyarrow not available - {item.name} requires pyarrow") + else: + raise + + +@pytest.hookimpl(hookwrapper=True) +def pytest_make_collect_report(collector): + """Wrap module collection to catch pyarrow import errors on Python 3.14. + + If we're on Python 3.14 and a test module raises ModuleNotFoundError + for 'pyarrow', mark the entire module as xfailed rather than failing collection. + + TODO(evertlammerts): Remove skip when pyarrow releases for 3.14. + https://github.com/duckdblabs/duckdb-internal/issues/6182 + """ + outcome = yield + report: pytest.CollectReport = outcome.get_result() + + if sys.version_info[:2] == (3, 14): + # Only handle failures from module collectors + if report.failed and collector.__class__.__name__ == "Module": + longreprtext = report.longreprtext + if "ModuleNotFoundError: No module named 'pyarrow'" in longreprtext: + report.outcome = "skipped" + reason = f"XFAIL: [pyarrow not available] {longreprtext}" + report.longrepr = (report.fspath, None, reason) + + # https://docs.pytest.org/en/latest/example/simple.html#control-skipping-of-tests-according-to-command-line-option # https://stackoverflow.com/a/47700320 def pytest_addoption(parser): diff --git a/tests/fast/api/test_duckdb_connection.py b/tests/fast/api/test_duckdb_connection.py index d197e639..4fa32749 100644 --- a/tests/fast/api/test_duckdb_connection.py +++ b/tests/fast/api/test_duckdb_connection.py @@ -4,7 +4,6 @@ from conftest import ArrowPandas, NumpyPandas import duckdb -import duckdb.typing pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/arrow/test_arrow_fetch_recordbatch.py b/tests/fast/arrow/test_arrow_fetch_recordbatch.py index 0070430b..a5804c87 100644 --- a/tests/fast/arrow/test_arrow_fetch_recordbatch.py +++ b/tests/fast/arrow/test_arrow_fetch_recordbatch.py @@ -234,7 +234,7 @@ def test_record_batch_reader_from_relation(self, duckdb_cursor): duckdb_cursor = duckdb.connect() duckdb_cursor.execute("CREATE table t as select range a from range(3000);") relation = duckdb_cursor.table("t") - record_batch_reader = relation.record_batch() + record_batch_reader = relation.fetch_record_batch() chunk = record_batch_reader.read_next_batch() assert len(chunk) == 3000 diff --git a/tests/fast/arrow/test_buffer_size_option.py b/tests/fast/arrow/test_buffer_size_option.py index 28e3bf58..c63adfcc 100644 --- a/tests/fast/arrow/test_buffer_size_option.py +++ b/tests/fast/arrow/test_buffer_size_option.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import VARCHAR +from duckdb.sqltypes import VARCHAR pa = pytest.importorskip("pyarrow") @@ -13,21 +13,21 @@ def test_arrow_buffer_size(self): # All small string res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.string() # All Large String con.execute("SET arrow_large_buffer_size=True") res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.large_string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.large_string() # All small string again con.execute("SET arrow_large_buffer_size=False") res = con.query("select 'bla'").fetch_arrow_table() assert res[0][0].type == pa.string() - res = con.query("select 'bla'").record_batch() + res = con.query("select 'bla'").fetch_record_batch() assert res.schema[0].type == pa.string() def test_arrow_buffer_size_udf(self): diff --git a/tests/fast/arrow/test_timestamp_timezone.py b/tests/fast/arrow/test_timestamp_timezone.py index 7e338626..27ddf3ac 100644 --- a/tests/fast/arrow/test_timestamp_timezone.py +++ b/tests/fast/arrow/test_timestamp_timezone.py @@ -64,7 +64,7 @@ def test_timestamp_stream(self, duckdb_cursor): con.execute("create table t (i timestamptz)") con.execute("insert into t values (NULL),('2021-11-15 02:30:00'::timestamptz)") rel = con.table("t") - arrow_tbl = rel.record_batch().read_all() + arrow_tbl = rel.fetch_record_batch().read_all() con.register("t2", arrow_tbl) assert con.execute("select * from t").fetchall() == con.execute("select * from t2").fetchall() diff --git a/tests/fast/test_expression.py b/tests/fast/test_expression.py index 5e61b455..c7eee6c1 100644 --- a/tests/fast/test_expression.py +++ b/tests/fast/test_expression.py @@ -13,7 +13,7 @@ LambdaExpression, StarExpression, ) -from duckdb.typing import INTEGER, TIMESTAMP, VARCHAR +from duckdb.sqltypes import INTEGER, TIMESTAMP, TINYINT, VARCHAR from duckdb.value.constant import IntegerValue, Value pytestmark = pytest.mark.skipif( @@ -804,7 +804,7 @@ def test_numeric_overflow(self): with pytest.raises(duckdb.OutOfRangeException, match="Overflow in multiplication of INT16"): rel2.fetchall() - val = duckdb.Value(100, duckdb.typing.TINYINT) + val = duckdb.Value(100, TINYINT) expr2 = ColumnExpression("salary") * val rel3 = rel.select(expr2) with pytest.raises(duckdb.OutOfRangeException, match="Overflow in multiplication of INT16"): diff --git a/tests/fast/test_parquet.py b/tests/fast/test_parquet.py index 3f6b1889..d418e1b9 100644 --- a/tests/fast/test_parquet.py +++ b/tests/fast/test_parquet.py @@ -3,9 +3,10 @@ import pytest import duckdb +import duckdb.sqltypes as duckdb_types -VARCHAR = duckdb.typing.VARCHAR -BIGINT = duckdb.typing.BIGINT +VARCHAR = duckdb_types.VARCHAR +BIGINT = duckdb_types.BIGINT filename = str(Path(__file__).parent / "data" / "binary_string.parquet") diff --git a/tests/fast/test_relation.py b/tests/fast/test_relation.py index f1b4b1fd..7b60a105 100644 --- a/tests/fast/test_relation.py +++ b/tests/fast/test_relation.py @@ -12,7 +12,7 @@ import duckdb from duckdb import ColumnExpression -from duckdb.typing import BIGINT, BOOLEAN, TINYINT, VARCHAR +from duckdb.sqltypes import BIGINT, BOOLEAN, TINYINT, VARCHAR @pytest.fixture(scope="session") diff --git a/tests/fast/test_replacement_scan.py b/tests/fast/test_replacement_scan.py index f0270167..57143b96 100644 --- a/tests/fast/test_replacement_scan.py +++ b/tests/fast/test_replacement_scan.py @@ -41,11 +41,11 @@ def fetch_arrow_table(rel): return rel.fetch_arrow_table() -def fetch_arrow_record_batch(rel): +def fetch_arrow_record_batch(rel: duckdb.DuckDBPyRelation): # Note: this has to executed first, otherwise we'll create a deadlock # Because it will try to execute the input at the same time as executing the relation # On the same connection (that's the core of the issue) - return rel.execute().record_batch() + return rel.execute().fetch_record_batch() def fetch_relation(rel): diff --git a/tests/fast/test_type.py b/tests/fast/test_type.py index 0eb96716..d8145166 100644 --- a/tests/fast/test_type.py +++ b/tests/fast/test_type.py @@ -4,8 +4,7 @@ import pytest import duckdb -import duckdb.typing -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BIT, BLOB, @@ -91,7 +90,7 @@ def test_incomplete_struct_type(self): with pytest.raises( duckdb.InvalidInputException, match="Could not convert empty dictionary to a duckdb STRUCT type" ): - duckdb.typing.DuckDBPyType({}) + DuckDBPyType({}) def test_map_type(self): type = duckdb.map_type(duckdb.sqltype("BIGINT"), duckdb.sqltype("DECIMAL(10, 2)")) @@ -228,26 +227,26 @@ def test_hash_method(self): # NOTE: we can support this, but I don't think going through hoops for an outdated version of python is worth it @pytest.mark.skipif(sys.version_info < (3, 9), reason="python3.7 does not store Optional[..] in a recognized way") def test_optional(self): - type = duckdb.typing.DuckDBPyType(Optional[str]) + type = DuckDBPyType(Optional[str]) assert type == "VARCHAR" - type = duckdb.typing.DuckDBPyType(Optional[Union[int, bool]]) + type = DuckDBPyType(Optional[Union[int, bool]]) assert type == "UNION(u1 BIGINT, u2 BOOLEAN)" - type = duckdb.typing.DuckDBPyType(Optional[list[int]]) + type = DuckDBPyType(Optional[list[int]]) assert type == "BIGINT[]" - type = duckdb.typing.DuckDBPyType(Optional[dict[int, str]]) + type = DuckDBPyType(Optional[dict[int, str]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) + type = DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) + type = DuckDBPyType(Optional[dict[Optional[int], Optional[str]]]) assert type == "MAP(BIGINT, VARCHAR)" - type = duckdb.typing.DuckDBPyType(Optional[Union[Optional[str], Optional[bool]]]) + type = DuckDBPyType(Optional[Union[Optional[str], Optional[bool]]]) assert type == "UNION(u1 VARCHAR, u2 BOOLEAN)" - type = duckdb.typing.DuckDBPyType(Union[str, None]) + type = DuckDBPyType(Union[str, None]) assert type == "VARCHAR" @pytest.mark.skipif(sys.version_info < (3, 10), reason="'str | None' syntax requires Python 3.10 or higher") def test_optional_310(self): - type = duckdb.typing.DuckDBPyType(str | None) + type = DuckDBPyType(str | None) assert type == "VARCHAR" def test_children_attribute(self): diff --git a/tests/fast/test_type_explicit.py b/tests/fast/test_type_explicit.py index 3b9fe334..f8a5ca63 100644 --- a/tests/fast/test_type_explicit.py +++ b/tests/fast/test_type_explicit.py @@ -1,4 +1,5 @@ import duckdb +import duckdb.sqltypes as duckdb_types class TestMap: @@ -9,7 +10,7 @@ def test_array_list_tuple_ambiguity(self): # By using an explicit duckdb.Value with an array type, we should convert the input as an array # and get an array (tuple) back - typ = duckdb.array_type(duckdb.typing.BIGINT, 2) + typ = duckdb.array_type(duckdb_types.BIGINT, 2) val = duckdb.Value((1, 2), typ) res = con.sql("SELECT $arg", params={"arg": val}).fetchall()[0][0] assert res == (1, 2) diff --git a/tests/fast/test_value.py b/tests/fast/test_value.py index 58aa7a4d..bf66bee8 100644 --- a/tests/fast/test_value.py +++ b/tests/fast/test_value.py @@ -6,7 +6,7 @@ import duckdb from duckdb import InvalidInputException, NotImplementedException -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BIT, BLOB, diff --git a/tests/fast/udf/test_null_filtering.py b/tests/fast/udf/test_null_filtering.py index e5c0d546..8bf2ce73 100644 --- a/tests/fast/udf/test_null_filtering.py +++ b/tests/fast/udf/test_null_filtering.py @@ -5,7 +5,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BLOB, BOOLEAN, @@ -24,6 +24,7 @@ UTINYINT, UUID, VARCHAR, + DuckDBPyType, ) pd = pytest.importorskip("pandas") @@ -31,7 +32,7 @@ class Candidate(NamedTuple): - type: duckdb.typing.DuckDBPyType + type: DuckDBPyType variant_one: Any variant_two: Any diff --git a/tests/fast/udf/test_remove_function.py b/tests/fast/udf/test_remove_function.py index 7ced339a..8ed739a4 100644 --- a/tests/fast/udf/test_remove_function.py +++ b/tests/fast/udf/test_remove_function.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import BIGINT, VARCHAR +from duckdb.sqltypes import BIGINT, VARCHAR pd = pytest.importorskip("pandas") pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/udf/test_scalar.py b/tests/fast/udf/test_scalar.py index 40e0d4de..80594c98 100644 --- a/tests/fast/udf/test_scalar.py +++ b/tests/fast/udf/test_scalar.py @@ -7,7 +7,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, BLOB, BOOLEAN, diff --git a/tests/fast/udf/test_scalar_arrow.py b/tests/fast/udf/test_scalar_arrow.py index 46f932a1..e3f18344 100644 --- a/tests/fast/udf/test_scalar_arrow.py +++ b/tests/fast/udf/test_scalar_arrow.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import BIGINT, INTEGER, VARCHAR +from duckdb.sqltypes import BIGINT, INTEGER, VARCHAR pd = pytest.importorskip("pandas") pa = pytest.importorskip("pyarrow") diff --git a/tests/fast/udf/test_scalar_native.py b/tests/fast/udf/test_scalar_native.py index 64ea5b5b..3c6b3cc8 100644 --- a/tests/fast/udf/test_scalar_native.py +++ b/tests/fast/udf/test_scalar_native.py @@ -1,7 +1,7 @@ import pytest import duckdb -from duckdb.typing import ( +from duckdb.sqltypes import ( BIGINT, HUGEINT, INTEGER, diff --git a/tests/pytest.ini b/tests/pytest.ini deleted file mode 100644 index 0c17afd5..00000000 --- a/tests/pytest.ini +++ /dev/null @@ -1,11 +0,0 @@ -# pytest.ini -[pytest] -filterwarnings = - error - ignore::UserWarning - ignore::DeprecationWarning - # Jupyter is throwing DeprecationWarnings - ignore:function ham\(\) is deprecated:DeprecationWarning - # Pyspark is throwing these warnings - ignore:distutils Version classes are deprecated:DeprecationWarning - ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning