diff --git a/.github/workflows/on_push.yml b/.github/disabled/on_push.yml similarity index 100% rename from .github/workflows/on_push.yml rename to .github/disabled/on_push.yml diff --git a/.github/workflows/cleanup_pypi.yml b/.github/workflows/cleanup_pypi.yml index c4300be3..e290faae 100644 --- a/.github/workflows/cleanup_pypi.yml +++ b/.github/workflows/cleanup_pypi.yml @@ -52,7 +52,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" - name: Run Cleanup env: diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index fdd2a838..ab696897 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -70,7 +70,7 @@ jobs: - name: Install Astral UV and enable the cache uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.9 enable-cache: true cache-suffix: -${{ github.workflow }} diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 7a4669cb..7e4b5107 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -14,9 +14,9 @@ on: - '!.github/workflows/on_push.yml' - '!.github/workflows/coverage.yml' -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true jobs: submodule_sanity_guard: diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 25cf3bdd..ea078376 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -52,9 +52,9 @@ on: required: false type: string -concurrency: - group: packaging-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: packaging-${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true defaults: run: @@ -63,6 +63,7 @@ defaults: jobs: build_sdist: name: Build an sdist and determine versions + if: ${{ github.event_name != 'pull_request' }} uses: ./.github/workflows/packaging_sdist.yml with: testsuite: all diff --git a/.github/workflows/packaging_sdist.yml b/.github/workflows/packaging_sdist.yml index 2723b437..87923f4c 100644 --- a/.github/workflows/packaging_sdist.yml +++ b/.github/workflows/packaging_sdist.yml @@ -58,7 +58,7 @@ jobs: - name: Install Astral UV uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" + version: "0.8.16" python-version: 3.11 - name: Build sdist diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index 4c7599a6..f99d6096 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -27,10 +27,12 @@ on: jobs: build_wheels: name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' + permissions: + actions: write # For GitHub Actions cache write access strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313 ] + python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, cp314t platform: - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } @@ -53,16 +55,31 @@ jobs: uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - + sccache --show-stats && + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs + SCCACHE_GHA_ENABLED: "on" + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} + ACTIONS_CACHE_SERVICE_V2: "1" + PYTHON_GIL: "1" + # Needed for 3.14t to indicate that it's OK to enable the GIL. Noop for other versions. steps: - name: Checkout DuckDB Python - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - + # The host doesn't need sccache, only the cibuildwheel container + # - name: Setup sccache # idea from https://github.com/pypa/cibuildwheel/issues/1030 + # uses: mozilla-actions/sccache-action@v0.0.9 + # with: + # version: "v0.10.0" + - name: Configure Cache Env + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Checkout DuckDB shell: bash if: ${{ inputs.duckdb-sha }} @@ -70,28 +87,57 @@ jobs: cd external/duckdb git fetch origin git checkout ${{ inputs.duckdb-sha }} - # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE if: ${{ inputs.set-version != '' }} - run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV - + run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }} SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }}" >> $GITHUB_ENV # Install Astral UV, which will be used as build-frontend for cibuildwheel - uses: astral-sh/setup-uv@v6 with: - version: "0.7.14" - enable-cache: false + version: "0.8.16" + enable-cache: true + activate-environment: true cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - - - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels - uses: pypa/cibuildwheel@v3.0 + python-version: ${{ + fromJSON('{"cp39":"3.9","cp310":"3.10","cp311":"3.11","cp312":"3.12","cp313":"3.13","cp314":"3.14","cp314t":"3.14t"}')[matrix.python] + }} + - name: Install cibuildwheel + run: | + echo "Installed uv version is ${{ steps.setup-uv.outputs.uv-version }}" + uv pip install cibuildwheel==3.1.4 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Windows) + if: matrix.platform.cibw_system == 'win' + shell: cmd + run: | + for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath`) do ( + echo Setting up x64 build environment... + call "%%i\VC\Auxiliary\Build\vcvars64.bat" + echo Verifying compiler paths: + where cl + echo VCINSTALLDIR=%VCINSTALLDIR% + echo Platform=%Platform% + uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + ) env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - + CIBW_ENVIRONMENT_WINDOWS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_VERBOSE_MAKEFILE=ON + CIBW_BUILD_VERBOSITY: 3 + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels (Non-Windows) + if: matrix.platform.cibw_system != 'win' + run: uv run --no-sync python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + CIBW_ENVIRONMENT_LINUX: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_ENVIRONMENT_MACOS: SKBUILD_BUILD_DIR=build/${{ matrix.python }}-${{ matrix.platform.arch }} CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" + CIBW_BUILD_VERBOSITY: 3 + CIBW_REPAIR_WHEEL_COMMAND_LINUX: | + sccache --show-stats + auditwheel repair -w {dest_dir} {wheel} - name: Upload wheel uses: actions/upload-artifact@v4 with: name: wheel-${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} path: wheelhouse/*.whl - compression-level: 0 + compression-level: 0 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6291b811..31413cab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,10 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "ipython", # used in duckdb.query_graph "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() - "pandas", # used for pandas dataframes all over the place - "pyarrow", # used for pyarrow support - "adbc_driver_manager", # for the adbc driver (TODO: this should live under the duckdb package) + "pandas; python_version < '3.14'", # used for pandas dataframes all over the place + "pandas>=2.3.0.dev0; python_version >= '3.14'", # nightly build for Python 3.14 + "pyarrow; python_version < '3.14'", # used for pyarrow support + "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] ###################################################################################################### @@ -204,6 +205,7 @@ required-environments = [ # ... but do always resolve for all of them "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'x86_64'", "python_version >= '3.9' and sys_platform == 'linux' and platform_machine == 'aarch64'", ] +prerelease = "allow" # for 3.14 # We just need pytorch for tests, wihtout GPU acceleration. PyPI doesn't host a cpu-only version for Linux, so we have # to configure the index url for cpu-only pytorch manually @@ -212,6 +214,11 @@ name = "pytorch-cpu" url = "https://download.pytorch.org/whl/cpu" explicit = true +[[tool.uv.index]] +name = "scientific-python-nightly" +url = "https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" +explicit = true + [tool.uv.sources] torch = [ { index = "pytorch-cpu" } ] torchvision = [ { index = "pytorch-cpu" } ] @@ -227,23 +234,27 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + # "pytest-randomly", # Ensures non-deterministic test order + # "pytest-xdist", # Parallel tests (separate processes) + # "pytest-run-parallel", # For free-threading testing "mypy", "coverage", - "gcovr", + "gcovr; python_version < '3.14'", "gcsfs", "packaging", - "polars", + "polars; python_version < '3.14'", "psutil", "py4j", "pyotp", - "pyspark", + "pyspark; python_version < '3.14'", "pytz", "requests", "urllib3", "fsspec>=2022.11.0", - "pandas>=2.0.0", - "pyarrow>=18.0.0", - "torch>=2.2.2; sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13'", + "pandas>=2.0.0; python_version < '3.14'", + "pandas>=2.3.0.dev0; python_version >= '3.14'", + "pyarrow>=18.0.0; python_version < '3.14'", + "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'linux' and platform_machine != 'aarch64' and python_version < '3.12'", "tensorflow-cpu>=2.14.0; sys_platform == 'win32' and python_version < '3.12'", @@ -256,10 +267,11 @@ scripts = [ # dependencies used for running scripts "ipython", "ipywidgets", "numpy", - "pandas", + "pandas; python_version < '3.14'", + "pandas>=2.3.0.dev0; python_version >= '3.14'", "pcpp", - "polars", - "pyarrow", + "polars; python_version < '3.14'", + "pyarrow; python_version < '3.14'", "pytz" ] pypi = [ # dependencies used by the pypi cleanup script @@ -305,6 +317,7 @@ filterwarnings = [ # Pyspark is throwing these warnings "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", + "ignore:ChainedAssignmentError.*:FutureWarning" ] [tool.coverage.run] @@ -379,12 +392,111 @@ manylinux-x86_64-image = "manylinux_2_28" manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" +enable = ["cpython-freethreading", "cpython-prerelease"] [tool.cibuildwheel.linux] -before-build = ["yum install -y ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +before-build = [ + "if [ \"$(uname -m)\" = \"aarch64\" ]; then ARCH=aarch64; else ARCH=x86_64; fi", + "curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz", + "cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin", + "which sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done", + "mkdir -p /root/.config/sccache && touch /root/.config/sccache/config", + "mkdir -p /tmp/sccache-cache", + "export SCCACHE_DIR=/tmp/sccache-cache", + "if [ -z \"$ACTIONS_RESULTS_URL\" ]; then echo \"ERROR: ACTIONS_RESULTS_URL not set in container\" && exit 1; fi", + "if [ -z \"$ACTIONS_RUNTIME_TOKEN\" ]; then echo \"ERROR: ACTIONS_RUNTIME_TOKEN not set in container\" && exit 1; fi", + "echo \"GitHub Actions environment variables verified in container\"", + "echo \"Testing sccache GitHub Actions cache access...\"", + "sccache --start-server", + "SCCACHE_LOG=debug sccache --show-stats 2>&1 | head -10 || true", + "echo 'int main(){return 0;}' > test.c", + "SCCACHE_LOG=debug sccache gcc -c test.c -o test.o 2>&1 | head -20 || true", + "sccache --show-stats", + "echo \"Sccache test complete\"" +] +before-test = "sccache --show-stats" [tool.cibuildwheel.macos] -before-build = ["brew install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +before-build = [ + "brew install sccache", + "echo 'Compiler launcher detection order check:'", + "which ccache || echo 'ccache not found'", + "which sccache || echo 'sccache not found'", + "echo 'PATH priority test:'", + "for prog in ccache sccache; do echo \"$prog: $(which $prog 2>/dev/null || echo 'not found')\"; done" +] [tool.cibuildwheel.windows] -before-build = ["choco install ccache"] +environment-pass = ["SCCACHE_GHA_ENABLED", "ACTIONS_RUNTIME_TOKEN", "ACTIONS_RESULTS_URL", "ACTIONS_CACHE_SERVICE_V2", "SCCACHE_C_CUSTOM_CACHE_BUSTER", "PYTHON_GIL"] +# if ccache is in the path, it's chosen before sccache +# and, uninstalling strawberry is much slower than just deleting the entry +before-build = [ + "del \"C:\\Strawberry\\c\\bin\\ccache.exe\"", + "choco install sccache", + "sccache --version" +] +before-test = [ + "sccache --show-stats" +] + +[[tool.scikit-build.overrides]] +# Regular Windows builds (non-free-threading) +if.platform-system = "^win32" +if.abi-flags = "" +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG", + "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=", + "-DCORE_EXTENSIONS=", + "-DBUILD_UNITTESTS=OFF" +] + +[[tool.scikit-build.overrides]] +# Free-threading Windows builds (Python 3.14t) +# To get sccache to work, needed a few things +# Ninja +# /Z7 so we don't write parallel PDBs +# The empty COMPILER_LAUNCHERS are because otherwise sccache is called twice (sccache sccache cl.exe ...) +# The LIBRARY is because it picks up python314.lib, instead of python314t.lib +# TODO: Dynamically add the "t" for future versions (ie: python315t) +if.platform-system = "^win32" +if.abi-flags = "t" +cmake.args = [ + "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=RelWithDebInfo", + "-DCMAKE_MSVC_DEBUG_INFORMATION_FORMAT=Embedded", + "-DCMAKE_C_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", + "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=/O2 /Z7 /DNDEBUG /DPY_NO_LINK_LIB /DPy_GIL_DISABLED", + "-DCMAKE_C_COMPILER_LAUNCHER=", + "-DCMAKE_CXX_COMPILER_LAUNCHER=", +] + # "-DCMAKE_SYSTEM_PROCESSOR=AMD64", + + #"-DCMAKE_SHARED_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + #"-DCMAKE_EXE_LINKER_FLAGS=/NODEFAULTLIB:python314.lib", + + # "-DCMAKE_LIBRARY_PATH=${Python3_LIBRARY_DIRS}", + + #"-DPython3_FIND_STRATEGY=LOCATION", + #"-DPython3_FIND_ABI=ANY;ANY;ANY;ON", + #"-DPython3_LIBRARY_RELEASE=python314t.lib", + # "-DCORE_EXTENSIONS=", + # "-DBUILD_UNITTESTS=OFF" + # "--debug-output", + # "--debug-find", + # "--debug-find-pkg=Python3", + # "--trace-expand", + # "--log-level=DEBUG" + diff --git a/test_mre.py b/test_mre.py new file mode 100644 index 00000000..99a2414b --- /dev/null +++ b/test_mre.py @@ -0,0 +1,11 @@ +import pandas as pd + + +def test_chained_assignment_warning(): + """This should trigger the ChainedAssignmentError warning we're trying to suppress""" + df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}) + + # This chained assignment triggers the warning + df['A'][0] = 999 + + assert df['A'][0] == 999 \ No newline at end of file diff --git a/test_warnings_mre.py b/test_warnings_mre.py new file mode 100644 index 00000000..f0eef295 --- /dev/null +++ b/test_warnings_mre.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +""" +Minimal reproducible example for pandas ChainedAssignmentError warning filtering +""" +import warnings +import pandas as pd + + +def test_chained_assignment_warning(): + """Test that reproduces the ChainedAssignmentError warning from pandas nightly""" + # This should trigger the warning we're trying to suppress + df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}) + + # This chained assignment should trigger the ChainedAssignmentError warning + df['A'][0] = 999 + + print("Test completed - if you see a FutureWarning above, the filter isn't working") + + +if __name__ == "__main__": + print("=== Running without any warning filters ===") + test_chained_assignment_warning() + + print("\n=== Running with PYTHONWARNINGS filter ===") + import os + os.environ['PYTHONWARNINGS'] = 'ignore:ChainedAssignmentError.*:FutureWarning' + test_chained_assignment_warning() + + print("\n=== Running with warnings.filterwarnings ===") + warnings.filterwarnings('ignore', message='ChainedAssignmentError.*', category=FutureWarning) + test_chained_assignment_warning() \ No newline at end of file diff --git a/tests/pytest.ini b/tests/pytest.ini index 0c17afd5..5081ee33 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,6 +2,8 @@ [pytest] filterwarnings = error + # Pandas ChainedAssignmentError warnings for 3.0 + ignore:ChainedAssignmentError.*:FutureWarning ignore::UserWarning ignore::DeprecationWarning # Jupyter is throwing DeprecationWarnings