diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml index c4300be3..e290faae 100644 --- a/.github/workflows/cleanup_pypi.yml +++ b/.github/workflows/cleanup_pypi.yml @@ -52,7 +52,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" - name: Run Cleanup env: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fdd2a838..ab696897 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -70,7 +70,7 @@ jobs: - name: Install Astral UV and enable the cache uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.9 enable-cache: true cache-suffix: -${{ github.workflow }} diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 7a4669cb..c8b01a59 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -14,9 +14,9 @@ on: - '!.github/workflows/on_push.yml' - '!.github/workflows/coverage.yml' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true jobs: submodule_sanity_guard: @@ -38,7 +38,7 @@ jobs: name: Run coverage tests needs: submodule_sanity_guard # Only run coverage test for draft PRs - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }} + if: false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }} uses: ./.github/workflows/coverage.yml with: duckdb_git_ref: ${{ github.base_ref }} diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 25cf3bdd..289aaa7c 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -52,9 +52,9 @@ on: required: false type: string -concurrency: - group: packaging-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: packaging-${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true defaults: run: @@ -63,6 +63,8 @@ defaults: jobs: build_sdist: name: Build an sdist and determine versions + if: false # Disable for dev + uses: ./.github/workflows/packaging_sdist.yml with: testsuite: all diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 2723b437..87923f4c 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -58,7 +58,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.11 - name: Build sdist diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4c7599a6..f4d094c9 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -27,17 +27,19 @@ on: jobs: build_wheels: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' + permissions: + actions: write # For GitHub Actions cache write access strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313 ] + python: [ cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t, cp314t platform: - - { os: windows-2025, arch: amd64, cibw_system: win } + #- { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } - - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + #- { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + #- { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-15, arch: universal2, cibw_system: macosx } + #- { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: @@ -45,6 +47,7 @@ jobs: - { minimal: true, python: cp311 } - { minimal: true, python: cp312 } - { minimal: true, platform: { arch: universal2 } } + - { python: cp314t, platform: { os: windows-2025 } } runs-on: ${{ matrix.platform.os }} env: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} @@ -53,16 +56,31 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - + sccache --show-stats && + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + SCCACHE_GHA_ENABLED: "on" + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} + ACTIONS_CACHE_SERVICE_V2: "1" + PYTHON_GIL: "1" + # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. steps: - name: Checkout DuckDB Python - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - + # The host doesn't need sccache, only the cibuildwheel container + # - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 + # uses: mozilla-actions/sccache-action@v0.0.9 + # with: + # version: "v0.10.0" + - name: Configure Cache Env + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -70,28 +88,55 @@ jobs: cd external/duckdb git fetch origin git checkout ${{ inputs.duckdb-sha }} - # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} - run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - + run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" - enable-cache: false + version: "0.8.16" + enable-cache: true + activate-environment: true cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.0 + python-version: ${{ matrix.python }} + - name: Install cibuildwheel + run: | + echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" + uv pip install cibuildwheel==3.1.4 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows) + if: matrix.platform.cibw_system == 'win' + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + echo Setting up x64 build environment... + call "%%i\VC\Auxiliary\Build\vcvars64.bat" + echo Verifying compiler paths: + where cl + echo VCINSTALLDIR=%VCINSTALLDIR% + echo Platform=%Platform% + uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + ) env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON + CIBW_BUILD_VERBOSITY: 3 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) + if: matrix.platform.cibw_system != 'win' + run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl - compression-level: 0 + compression-level: 0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6291b811..b5af4846 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() "pandas", # used for pandas dataframes all over the place - "pyarrow", # used for pyarrow support - "adbc_driver_manager", # for the adbc driver (TODO: this should live under the duckdb package) + "pyarrow; python_version < '3.14'", # used for pyarrow support + "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] ###################################################################################################### @@ -204,6 +204,7 @@ required-environments = [ # ... but do always resolve for all of them "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'x86_64'", "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'aarch64'", ] +prerelease = "allow" # for 3.14 # We just need pytorch for tests, wihtout GPU acceleration. PyPI doesn't host a cpu-only version for Linux, so we have # to configure the index url for cpu-only pytorch manually @@ -227,23 +228,26 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + # "pytest-randomly", # Ensures non-deterministic test order + # "pytest-xdist", # Parallel tests (separate processes) + # "pytest-run-parallel", # For free-threading testing "mypy", "coverage", - "gcovr", + "gcovr; python_version < '3.14'", "gcsfs", "packaging", - "polars", + "polars; python_version < '3.14'", "psutil", "py4j", "pyotp", - "pyspark", + "pyspark; python_version < '3.14'", "pytz", "requests", "urllib3", "fsspec>=2022.11.0", "pandas>=2.0.0", - "pyarrow>=18.0.0", - "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", + "pyarrow>=18.0.0; python_version < '3.14'", + "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'", @@ -258,8 +262,8 @@ scripts = [ # dependencies used for running scripts "numpy", "pandas", "pcpp", - "polars", - "pyarrow", + "polars; python_version < '3.14'", + "pyarrow; python_version < '3.14'", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script @@ -305,6 +309,7 @@ filterwarnings = [ # Pyspark is throwing these warnings "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", + "ignore:ChainedAssignmentError.*:FutureWarning" ] [tool.coverage.run] @@ -379,12 +384,113 @@ manylinux-x86_64-image = "manylinux_2_28" manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" +enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -before-build = ["yum install -y ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +before-build = [ + "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi", + "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz", + "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin", + "which sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done", + "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config", + "mkdir -p /tmp/sccache-cache", + "export SCCACHE_DIR=/tmp/sccache-cache", + "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", + "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", + "echo \"GitHub Actions environment variables verified in container\"", + "echo \"Testing sccache GitHub Actions cache access...\"", + "sccache --start-server", + "SCCACHE_LOG=debug sccache --show-stats 2>&1 | head -10 || true", + "echo 'int main(){return 0;}' > test.c", + "SCCACHE_LOG=debug sccache gcc -c test.c -o test.o 2>&1 | head -20 || true", + "sccache --show-stats", + "echo \"Sccache test complete\"" +] +before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -before-build = ["brew install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +before-build = [ + "brew install sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done" +] [tool.cibuildwheel.windows] -before-build = ["choco install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +# if ccache is in the path, it's chosen before sccache +# and, uninstalling strawberry is much slower than just deleting the entry +before-build = [ + "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", + "choco install sccache", + "sccache --version" +] +before-test = [ + "sccache --show-stats" +] + +[[tool.scikit-build.overrides]] +# Regular Windows builds (non-free-threading) +if.platform-system = "^win32" +if.abi-flags = "" +inherit.cmake.define = "append" +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/Z7", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/Z7", + "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DCORE_EXTENSIONS=", + "-DBUILD_UNITTESTS=OFF" +] + +[[tool.scikit-build.overrides]] +# Free-threading Windows builds (Python 3.14t) +# To get sccache to work, needed a few things +# Ninja +# /Z7 so we don't write parallel PDBs +# The empty COMPILER_LAUNCHERS are because otherwise sccache is called twice (sccache sccache cl.exe ...) +# The LIBRARY is because it picks up python314.lib, instead of python314t.lib +# TODO: Dynamically add the "t" for future versions (ie: python315t) +if.platform-system = "^win32" +if.abi-flags = "t" +inherit.cmake.define = "append" +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/Z7 /DPy_GIL_DISABLED", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/Z7 /DPy_GIL_DISABLED", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=", +] + # "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + + #"-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + #"-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + + # "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", + + #"-DPython3_FIND_STRATEGY=LOCATION", + #"-DPython3_FIND_ABI=ANY;ANY;ANY;ON", + #"-DPython3_LIBRARY_RELEASE=python314t.lib", + # "-DCORE_EXTENSIONS=", + # "-DBUILD_UNITTESTS=OFF" + # "--debug-output", + # "--debug-find", + # "--debug-find-pkg=Python3", + # "--trace-expand", + # "--log-level=DEBUG" + diff --git a/tests/pytest.ini b/tests/pytest.ini index 0c17afd5..5081ee33 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,6 +2,8 @@ [pytest] filterwarnings = error + # Pandas ChainedAssignmentError warnings for 3.0 + ignore:ChainedAssignmentError.*:FutureWarning ignore::UserWarning ignore::DeprecationWarning # Jupyter is throwing DeprecationWarnings