diff --git a/.github/workflows/on_pr.yml b/.github/workflows/on_pr.yml index 7a4669cb..e22fc905 100644 --- a/.github/workflows/on_pr.yml +++ b/.github/workflows/on_pr.yml @@ -2,8 +2,7 @@ name: Tests and builds on PR on: pull_request: branches: - - main - - v*.*-* + - '**' types: [opened, reopened, ready_for_review, converted_to_draft, synchronize] paths-ignore: - '**.md' @@ -13,10 +12,11 @@ on: - '.github//**' - '!.github/workflows/on_push.yml' - '!.github/workflows/coverage.yml' + workflow_dispatch: -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true jobs: submodule_sanity_guard: @@ -38,7 +38,7 @@ jobs: name: Run coverage tests needs: submodule_sanity_guard # Only run coverage test for draft PRs - if: ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }} + if: false # ${{ github.event_name == 'pull_request' && github.event.pull_request.draft == true }} uses: ./.github/workflows/coverage.yml with: duckdb_git_ref: ${{ github.base_ref }} diff --git a/.github/workflows/on_push.yml b/.github/workflows/on_push.yml index 1a282d69..adbdcac4 100644 --- a/.github/workflows/on_push.yml +++ b/.github/workflows/on_push.yml @@ -1,6 +1,6 @@ name: Tests and coverage on push on: - push: + workflow_dispatch: branches-ignore: - main - v*.*-* diff --git a/.github/workflows/packaging.yml b/.github/workflows/packaging.yml index 16771deb..b169da2b 100644 --- a/.github/workflows/packaging.yml +++ b/.github/workflows/packaging.yml @@ -52,9 +52,9 @@ on: required: false type: string -concurrency: - group: packaging-${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true +# concurrency: +# group: packaging-${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true defaults: run: @@ -63,6 +63,7 @@ defaults: jobs: build_sdist: name: Build an sdist and determine versions + if: false # disable for dev uses: ./.github/workflows/packaging_sdist.yml with: testsuite: all diff --git a/.github/workflows/packaging_wheels.yml b/.github/workflows/packaging_wheels.yml index ea13b674..b38f4043 100644 --- a/.github/workflows/packaging_wheels.yml +++ b/.github/workflows/packaging_wheels.yml @@ -30,14 +30,14 @@ jobs: strategy: fail-fast: false matrix: - python: [ cp39, cp310, cp311, cp312, cp313, cp314, cp314t ] + python: [ cp313, cp314t ] # cp39, cp310, cp311, cp312, cp313, cp314, platform: - - { os: windows-2025, arch: amd64, cibw_system: win } + # - { os: windows-2025, arch: amd64, cibw_system: win } - { os: ubuntu-24.04, arch: x86_64, cibw_system: manylinux } - - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } - - { os: macos-15, arch: arm64, cibw_system: macosx } - - { os: macos-15, arch: universal2, cibw_system: macosx } - - { os: macos-13, arch: x86_64, cibw_system: macosx } + # - { os: ubuntu-24.04-arm, arch: aarch64, cibw_system: manylinux } + # - { os: macos-15, arch: arm64, cibw_system: macosx } + # - { os: macos-15, arch: universal2, cibw_system: macosx } + # - { os: macos-13, arch: x86_64, cibw_system: macosx } minimal: - ${{ inputs.minimal }} exclude: @@ -54,27 +54,36 @@ jobs: CIBW_TEST_SKIP: ${{ inputs.testsuite == 'none' && '*' || '*-macosx_universal2' }} CIBW_TEST_SOURCES: tests CIBW_BEFORE_TEST: > - uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} && + sccache --show-stats && + uv export --only-group test --no-emit-project --output-file pylock.toml --directory {project} --quiet && uv pip install -r pylock.toml CIBW_TEST_COMMAND: > - uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs - + uv run -v pytest ${{ inputs.testsuite == 'fast' && './tests/fast' || './tests' }} --verbose --ignore=./tests/stubs --durations=5 + SCCACHE_GHA_ENABLED: "on" + SCCACHE_C_CUSTOM_CACHE_BUSTER: ${{ toJSON(matrix) }} + ACTIONS_CACHE_SERVICE_V2: "1" + PYTHON_GIL: "1" steps: - name: Checkout DuckDB Python - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: ref: ${{ inputs.duckdb-python-sha }} fetch-depth: 0 submodules: true - - name: Checkout DuckDB shell: bash + continue-on-error: true # needed when we're merging into a branch if: ${{ inputs.duckdb-sha }} run: | cd external/duckdb git fetch origin git checkout ${{ inputs.duckdb-sha }} - + - name: Configure Cache Env + uses: actions/github-script@v7 + with: + script: | + core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); # Make sure that OVERRIDE_GIT_DESCRIBE is propagated to cibuildwhel's env, also when it's running linux builds - name: Set OVERRIDE_GIT_DESCRIBE shell: bash @@ -94,9 +103,21 @@ jobs: env: CIBW_ARCHS: ${{ matrix.platform.arch == 'amd64' && 'AMD64' || matrix.platform.arch }} CIBW_BUILD: ${{ matrix.python }}-${{ matrix.platform.cibw_system }}_${{ matrix.platform.arch }} - # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds. - # TODO: Remove PYTHON_GIL=1 when free-threaded is supported. - CIBW_ENVIRONMENT: PYTHON_GIL=1 + # PYTHON_GIL=1: Suppresses the RuntimeWarning that the GIL is enabled on free-threaded builds. + # TODO: Remove PYTHON_GIL=1 when free-threaded is supported. + CIBW_ENVIRONMENT: PYTHON_GIL=1 CMAKE_C_COMPILER_LAUNCHER="" CMAKE_CXX_COMPILER_LAUNCHER="" CFLAGS="-Wno-attributes" CXXFLAGS="-Wno-attributes" + CIBW_ENVIRONMENT_PASS_LINUX: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL + CIBW_ENVIRONMENT_PASS_MACOS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL + CIBW_ENVIRONMENT_PASS_WINDOWS: SCCACHE_GHA_ENABLED ACTIONS_RUNTIME_TOKEN ACTIONS_RESULTS_URL ACTIONS_CACHE_SERVICE_V2 SCCACHE_C_CUSTOM_CACHE_BUSTER PYTHON_GIL + CIBW_BEFORE_BUILD_LINUX: > + if [ "$(uname -m)" = "aarch64" ]; then ARCH=aarch64; else ARCH=x86_64; fi && + curl -L https://github.com/mozilla/sccache/releases/download/v0.10.0/sccache-v0.10.0-${ARCH}-unknown-linux-musl.tar.gz | tar xz && + cp sccache-v0.10.0-${ARCH}-unknown-linux-musl/sccache /usr/bin && + sccache --show-stats + CIBW_BEFORE_BUILD_MACOS: brew install sccache + CIBW_BEFORE_BUILD_WINDOWS: > + del "C:\Strawberry\c\bin\ccache.exe" && + choco install sccache - name: Upload wheel uses: actions/upload-artifact@v4 with: diff --git a/CMakeLists.txt b/CMakeLists.txt index a9bc047d..2975f37f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,8 +46,13 @@ else() endif() duckdb_add_library(duckdb_target) +# Use unity for the external duckdb_target +set_target_properties(duckdb_target PROPERTIES UNITY_BUILD ON UNITY_BUILD_BATCH_SIZE 32) + + # Bundle in INTERFACE library add_library(_duckdb_dependencies INTERFACE) + target_link_libraries(_duckdb_dependencies INTERFACE pybind11::pybind11 duckdb_target @@ -77,6 +82,9 @@ pybind11_add_module(_duckdb $ $ ) +# don't use unity for the duckdb-python code - we just want one file per target +set_target_properties(_duckdb PROPERTIES UNITY_BUILD OFF) + # add _duckdb_dependencies target_link_libraries(_duckdb PRIVATE _duckdb_dependencies) diff --git a/pyproject.toml b/pyproject.toml index bcbb24f6..444a2d0b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -121,7 +121,7 @@ cmake.build-type = "Debug" if.state = "editable" if.env.COVERAGE = false inherit.cmake.define = "append" -cmake.define.DISABLE_UNITY = "1" +# cmake.define.DISABLE_UNITY = "1" [tool.scikit-build.sdist] include = [ @@ -227,6 +227,7 @@ test = [ # dependencies used for running tests "pytest", "pytest-reraise", "pytest-timeout", + "pytest-xdist", "mypy", "coverage", "gcovr; python_version < '3.14'", @@ -380,12 +381,3 @@ manylinux-pypy_x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" manylinux-pypy_aarch64-image = "manylinux_2_28" enable = ["cpython-freethreading", "cpython-prerelease"] - -[tool.cibuildwheel.linux] -before-build = ["yum install -y ccache"] - -[tool.cibuildwheel.macos] -before-build = ["brew install ccache"] - -[tool.cibuildwheel.windows] -before-build = ["choco install ccache"]