diff --git a/.github/workflows/dev_testing.yml b/.github/workflows/dev_testing.yml new file mode 100644 index 00000000..f0c9e4cd --- /dev/null +++ b/.github/workflows/dev_testing.yml @@ -0,0 +1,225 @@ +name: On Push Fast Tests +on: push + +# This workflow is runnable locally for debugging: +# using https://github.com/nektos/act +# +jobs: + build_wheels: + name: 'Wheel: ${{ matrix.python }}-${{ matrix.platform.os }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }}' + strategy: + fail-fast: false + matrix: + python: [ cp39, cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314t, + platform: + #- { os: windows-2025, arch: amd64, cibw_system: win } + - { os: ubuntu-latest, arch: x86_64, cibw_system: manylinux } + # - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + #- { os: macos-15, arch: arm64, cibw_system: macosx } + #- { os: macos-26, arch: arm64, cibw_system: macosx } + # - { os: macos-15, arch: universal2, cibw_system: macosx } + # - { os: macos-13, arch: x86_64, cibw_system: macosx } + #exclude: + # - { python: cp314t, platform: { os: windows-2025 } } + + runs-on: ${{ matrix.platform.os }} + env: + CIBW_TEST_SOURCES: tests + CIBW_BEFORE_TEST_LINUX: > + sccache --show-stats && + uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet && + uv pip install -r pylock.toml + CIBW_BEFORE_TEST_MACOS: > + sccache --show-stats && + uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet && + uv pip install -r pylock.toml + CIBW_BEFORE_TEST_WINDOWS: > + sccache --show-stats && + uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet && + uv pip install -r pylock.toml + CIBW_TEST_COMMAND: > + uv run -v pytest ./tests/fast --verbose --ignore=./tests/stubs --durations=5 -n 2 + # && uv run -v pytest ./tests/fast/threading --durations=5 --parallel-threads=10 --iterations=5 -n 2 + + CMAKE_VERBOSE_MAKEFILE: "ON" + + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} + SCCACHE_BASEDIR: "/tmp/duckdb-build" + ACTIONS_CACHE_SERVICE_V2: "1" + + TMPDIR: "/tmp/duckdb-build" + TEMP: "/tmp/duckdb-build" + steps: + - name: Set SCCACHE_GHA_ENABLED + run: echo "SCCACHE_GHA_ENABLED=${{ env.ACT == 'true' && 'off' || 'on' }}" >> $GITHUB_ENV + - name: Checkout DuckDB Python + if: ${{ env.ACT != 'true' }} + uses: actions/checkout@v5 + with: + # ref: ${{ inputs.duckdb-python-sha }} + fetch-depth: 0 + submodules: true + - name: Checkout DuckDB + if: ${{ env.ACT != 'true' && inputs['duckdb-sha'] != '' }} + shell: bash + continue-on-error: true # sometimes helpful when merging branch->branch + run: | + cd external/duckdb + git fetch origin + git checkout ${{ inputs.duckdb-sha }} + - name: Configure Cache Env + # if: ${{ env.ACT != 'true' }} + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds + - name: Set OVERRIDE_GIT_DESCRIBE + shell: bash + if: ${{ inputs.set-version != '' }} + run: echo "CIBW_ENVIRONMENT=OVERRIDE_GIT_DESCRIBE=${{ inputs.set-version }}" >> $GITHUB_ENV + + # # Install Astral UV, which will be used as build-frontend for cibuildwheel + - uses: astral-sh/setup-uv@v6 + if: ${{ env.ACT != 'true' }} + with: + # version: "0.8.19" # Version is important - it determines which Python build you get. Bump when new version of Python releases. + enable-cache: false + activate-environment: true + cache-suffix: -${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + python-version: ${{ matrix.python }} + + # Used when running workflows locally + - name: Install UV + if: ${{ env.ACT == 'true' }} + shell: bash + env: + TMPDIR: /tmp + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "$HOME/.local/bin" >> $GITHUB_PATH + export PATH="$HOME/.local/bin:$PATH" + uv --version + + - name: Install sccache + if: ${{ matrix.platform.cibw_system == 'win' }} + shell: bash + # TODO: Find a better PATH dir to put this in + run: | + curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-x86_64-pc-windows-msvc.zip -o sccache.zip + unzip sccache.zip + cp sccache-*/sccache.exe "C:/Strawberry/c/bin/sccache.exe" + sccache --version + + # Load MSVC environment, needed for Windows - ninja builds + - uses: ilammy/msvc-dev-cmd@v1 + if: ${{ matrix.platform.cibw_system == 'win' }} + with: + arch: ${{ matrix.platform.arch }} + + - name: Build${{ inputs.testsuite != 'none' && ' and test ' || ' ' }}wheels + uses: pypa/cibuildwheel@v3.2 + env: + CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} + CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} + + #CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation" + UV_PROJECT_ENVIRONMENT: /project/.venv + #UV_NO_BUILD_ISOLATION: "true" + CMAKE_C_COMPILER_LAUNCHER: "" + CMAKE_CXX_COMPILER_LAUNCHER: "" + # CFLAGS: "-Wno-attributes" + # CXXFLAGS: "-Wno-attributes" + # UV_PYTHON: ${{ matrix.python }} + PYTHON_GIL: 1 + CIBW_BUILD_VERBOSITY: 3 + # SCCACHE Notes: + # - Without the /project dir and tmpdirs (not sure exactly which mattered), the python environment is installed to a random tmp dir, which breaks + # the cache key... so, only the external/duckdb would cache, and not the project itself + # - GHA cache is limited to 10GB LRU. + # - SCCACHE_BASEDIR is not implemented in sccache (https://github.com/mozilla/sccache/issues/35) + # - Using -fdebug-prefix-map/-fmacro-prefix-map for path normalization instead (Mozilla Firefox approach) + # - Mozilla bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1524662 "Support gcc/clang-like build path prefix map" + # - UV creates random build-env-* dirs causing cache misses (https://github.com/astral-sh/uv/issues/13096) + # - Using --no-isolation to eliminate random UV build environments + # + # SCCACHE_LOG: trace + # SCCACHE_LOG_LEVEL: trace + # RUST_LOG: trace + # SCCACHE_NO_DAEMON: "1" + # PYTHONPATH="/project" ("." on mac and windows) + + # UV_NO_BUILD_ISOLATION="true" + # no-build-isolation uses the same build-env path for each build, stable paths needed for caching + # SCCACHE_BASEDIR="/tmp/duckdb-build" + # TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build" + # UV_PROJECT_ENVIRONMENT="/project/.venv" + + #SCCACHE_BASEDIR="/tmp/duckdb-build" + #TMPDIR="/tmp/duckdb-build" TEMP="/tmp/duckdb-build" + #UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/.venv" + + # SCCACHE_BASEDIR="/tmp/duckdb-build" + # TMPDIR="C:/tmp/duckdb-build" TEMP="C:/tmp/duckdb-build" + # UV_PROJECT_ENVIRONMENT="C:/project/.venv" + #CMAKE_BUILD_TYPE=RelWithDebInfo + CIBW_ENVIRONMENT_WINDOWS: > + SCCACHE_GHA_ENABLED=true + CMAKE_BUILD_TYPE=Release + PYTHONPATH="." + TMPDIR="C:/tmp/duckdb-build" + TEMP="C:/tmp/duckdb-build" + CMAKE_ARGS="-DPython3_EXECUTABLE=C:/tmp/duckdb-build/stable-venv/Scripts/python.exe -DPython_EXECUTABLE=C:/tmp/duckdb-build/stable-venv/Scripts/python.exe -DCMAKE_PREFIX_PATH=C:/tmp/duckdb-build/stable-venv" + CIBW_ENVIRONMENT_MACOS: > + SCCACHE_GHA_ENABLED=true + PYTHONPATH="." + TMPDIR="/tmp/duckdb-build" + TEMP="/tmp/duckdb-build" + UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/.venv" + CIBW_BUILD_DIR="/tmp/duckdb-build/build" + PATH="/tmp/duckdb-build/stable-venv/bin:$PATH" + CMAKE_ARGS="-DPython3_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DPython_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DCMAKE_PREFIX_PATH=/tmp/duckdb-build/stable-venv" + CIBW_ENVIRONMENT_LINUX: > + SCCACHE_GHA_ENABLED=true + PYTHONPATH="/project" + TMPDIR="/tmp/duckdb-build" + TEMP="/tmp/duckdb-build" + UV_PROJECT_ENVIRONMENT="/tmp/duckdb-build/stable-venv" + CMAKE_ARGS="-DPython3_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DPython_EXECUTABLE=/tmp/duckdb-build/stable-venv/bin/python -DCMAKE_PREFIX_PATH=/tmp/duckdb-build/stable-venv" + + CIBW_ENVIRONMENT_PASS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER SCCACHE_LOG SCCACHE_LOG_LEVEL SCCACHE_NO_DAEMON RUST_LOG SCCACHE_BASEDIR TMPDIR TEMP PIP_CACHE_DIR UV_PROJECT_ENVIRONMENT PYTHONPATH UV_NO_BUILD_ISOLATION UV_CACHE_DIR UV_PYTHON CMAKE_C_COMPILER_LAUNCHER CMAKE_CXX_COMPILER_LAUNCHER CMAKE_VERBOSE_MAKEFILE CIBW_BUILD_FRONTEND PYTHON_GIL CIBW_BUILD_VERBOSITY CIBW_BUILD_DIR VIRTUAL_ENV CMAKE_ARGS + + CIBW_BEFORE_BUILD_LINUX: > + if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi && + curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz && + cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin && + sccache --show-stats && + mkdir -p /tmp/duckdb-build /tmp/pip-cache && + uv venv /tmp/duckdb-build/stable-venv --python "$(which python)" && + uv pip install --python /tmp/duckdb-build/stable-venv/bin/python 'pybind11[global]>=2.6.0' + CIBW_BEFORE_BUILD_MACOS: > + mkdir -p /tmp/duckdb-build /tmp/duckdb-build/uv-cache /tmp/pip-cache && + brew install sccache && + rm -f /tmp/duckdb-build/stable-venv && + ln -sf "$VIRTUAL_ENV" /tmp/duckdb-build/stable-venv && + uv pip install scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10' + + # Build dependencies installs needed if using no-isolation + # uv pip install scikit-build-core 'pybind11[global]>=2.6.0' setuptools-scm 'cmake>=3.29.0' 'ninja>=1.10' && + CIBW_BEFORE_BUILD_WINDOWS: > + (mkdir "C:\tmp\duckdb-build" 2>nul & mkdir "C:\tmp\duckdb-build\uv-cache" 2>nul & mkdir "C:\tmp\pip-cache" 2>nul) && + del "C:\Strawberry\c\bin\ccache.exe" 2>nul && + sccache --show-stats && + mklink /j "C:\tmp\duckdb-build\stable-venv" "%VIRTUAL_ENV%" && + call "C:\tmp\duckdb-build\stable-venv\Scripts\activate.bat" && + uv pip install "pybind11[global]>=2.6.0" + + # && + # uv pip install scikit-build-core "pybind11[global]>=2.6.0" setuptools-scm "cmake>=3.29.0" "ninja>=1.10" + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.python }}-${{ matrix.platform.os }}_${{ matrix.platform.arch }} + path: wheelhouse/*.whl + compression-level: 0 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index a9bc047d..abda18e8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,10 @@ else() endif() duckdb_add_library(duckdb_target) +# Use unity for the external duckdb_target +set_target_properties(duckdb_target PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 32) + + # Bundle in INTERFACE library add_library(_duckdb_dependencies INTERFACE) target_link_libraries(_duckdb_dependencies INTERFACE @@ -77,6 +81,9 @@ pybind11_add_module(_duckdb $ $ ) +# don't use unity for the duckdb-python code - we just want one file per target +set_target_properties(_duckdb PROPERTIES UNITY_BUILD OFF) + # add _duckdb_dependencies target_link_libraries(_duckdb PRIVATE _duckdb_dependencies) diff --git a/pyproject.toml b/pyproject.toml index bcbb24f6..12af32ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,7 @@ all = [ # users can install duckdb with 'duckdb[all]', which will install this l "ipython", # used in duckdb.query_graph "fsspec", # used in duckdb.filesystem "numpy", # used in duckdb.experimental.spark and in duckdb.fetchnumpy() - "pandas; python_version < '3.14'", # used for pandas dataframes all over the place + "pandas", # used for pandas dataframes all over the place "pyarrow; python_version < '3.14'", # used for pyarrow support "adbc_driver_manager; python_version < '3.14'", # for the adbc driver (TODO: this should live under the duckdb package) ] @@ -73,6 +73,7 @@ cmake.version = ">=3.29.0" ninja.version = ">=1.10" ninja.make-fallback = false metadata.version.provider = "scikit_build_core.metadata.setuptools_scm" +#build.tool-args = ["-v", "-d", "stats"] [tool.scikit-build.wheel] cmake = true @@ -123,6 +124,39 @@ if.env.COVERAGE = false inherit.cmake.define = "append" cmake.define.DISABLE_UNITY = "1" +[[tool.scikit-build.overrides]] +if.platform-system = "^win32" +inherit.cmake.define = "append" + +# Verbose Makefile breaks sccacheing +cmake.define.CMAKE_VERBOSE_MAKEFILE = "OFF" +cmake.define.CMAKE_C_FLAGS_INIT="/Z7" +cmake.define.CMAKE_CXX_FLAGS_INIT="/Z7" +cmake.define.CMAKE_MSVC_DEBUG_INFORMATION_FORMAT="Embedded" +#build.tool-args = ["/m:4"] +cmake.args = [ + "-G", "Ninja", + "--log-level=DEBUG", + "-DCMAKE_BUILD_TYPE=Release" +] + +[[tool.scikit-build.overrides]] +# Windows Free-Threading +if.platform-system = "^win32" +if.abi-flags = "t" +inherit.cmake.define = "append" +# cmake.build-type = "RelWithDebInfo" +# /EHsc: Solution for many of the crashes observed on windows +# /Z7 / Embedded: Needed for sccache +cmake.define.CMAKE_MSVC_DEBUG_INFORMATION_FORMAT="Embedded" +cmake.define.CMAKE_C_FLAGS_INIT="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED" +cmake.define.CMAKE_CXX_FLAGS_INIT="/Z7 /DPy_MOD_GIL_USED /DPy_GIL_DISABLED" + +#build.tool-args = ["/m:4"] +#, "/verbosity:diagnostic", "/property:UseCommandLineEcho=true"] + +# for msbuild + [tool.scikit-build.sdist] include = [ "README.md", @@ -219,7 +253,7 @@ torchvision = [ { index = "pytorch-cpu" } ] [dependency-groups] # used for development only, requires pip >=25.1.0 stubdeps = [ # dependencies used for typehints in the stubs "fsspec", - "pandas; python_version < '3.14'", + "pandas", "polars; python_version < '3.14'", "pyarrow; python_version < '3.14'", ] @@ -227,6 +261,8 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + "pytest-xdist", # multi-processed tests, if `-n | auto` + "pytest-randomly", # randomizes test order to ensure no test dependencies, enabled on install "mypy", "coverage", "gcovr; python_version < '3.14'", @@ -241,7 +277,7 @@ test = [ # dependencies used for running tests "requests", "urllib3", "fsspec>=2022.11.0", - "pandas>=2.0.0; python_version < '3.14'", + "pandas>=2.0.0", "pyarrow>=18.0.0; python_version < '3.14'", "torch>=2.2.2; python_version < '3.14' and (sys_platform != 'darwin' or platform_machine != 'x86_64' or python_version < '3.13')", "tensorflow==2.14.0; sys_platform == 'darwin' and python_version < '3.12'", @@ -256,7 +292,7 @@ scripts = [ # dependencies used for running scripts "ipython", "ipywidgets", "numpy", - "pandas; python_version < '3.14'", + "pandas", "pcpp", "polars; python_version < '3.14'", "pyarrow; python_version < '3.14'", @@ -306,6 +342,7 @@ filterwarnings = [ "ignore:distutils Version classes are deprecated:DeprecationWarning", "ignore:is_datetime64tz_dtype is deprecated:DeprecationWarning", ] +timeout = 600 # don't let individual tests "hang" [tool.coverage.run] branch = true @@ -388,4 +425,5 @@ before-build = ["yum install -y ccache"] before-build = ["brew install ccache"] [tool.cibuildwheel.windows] -before-build = ["choco install ccache"] +# Strawberry ccache is already installed on the cibuildwheel images +before-build = [] diff --git a/util/localbuild.sh b/util/localbuild.sh new file mode 100755 index 00000000..c4626f86 --- /dev/null +++ b/util/localbuild.sh @@ -0,0 +1,33 @@ +#export UV_PYTHON=3.14.0rc3+freethreaded +export UV_PYTHON=3.12 +export UV_PROJECT_ENVIRONMENT=.venv_$UV_PYTHON + +if [ ! -d $UV_PROJECT_ENVIRONMENT ]; then + uv venv +else + echo $UV_PROJECT_ENVIRONMENT exists +fi + +source $UV_PROJECT_ENVIRONMENT/bin/activate +uv sync --no-install-project + +export CCACHE_PROGRAM=ccache +export PYTHON_GIL=1 +# export UV_CACHE_DIR="$(pwd)/.cache" +export CCACHE_BASEDIR=/home/ec2-user/git/duckdb-pythonf +#export CCACHE_NOHASHDIR=1 +#export UV_NO_BUILD_ISOLATION=true +#export UV_NO_EDITABLE=true +#export SKBUILD_EDITABLE_MODE=redirect +export UV_BUILD_DIR=/home/ec2-user/git/duckdb-pythonf/build +export CMAKE_BUILD_DIR="/home/ec2-user/git/duckdb-pythonf/build" +export SKBUILD_BUILD_DIR="/home/ec2-user/git/duckdb-pythonf/build" +export SKBUILD_CMAKE_ARGS="-B/home/ec2-user/git/duckdb-pythonf/build" + + +export CMAKE_ARGS="-DPython3_EXECUTABLE=$UV_PROJECT_ENVIRONMENT/bin/python -DPython_EXECUTABLE=$UV_PROJECT_ENVIRONMENT/bin/python -DCMAKE_PREFIX_PATH=$UV_PROJECT_ENVIRONMENT" + +uv sync --no-build-isolation -vv --reinstall + +# uv sync --no-build-isolation --reinstall -vv --no-editable +